Read data records from a JSON dump of inputs. Avoids command line flattening of records.
358297
def _read_from_cwlinput(in_file, work_dir, runtime, parallel, input_order, output_cwl_keys):
    
    with open(in_file) as in_handle:
        inputs = json.load(in_handle)
    items_by_key = {}
    input_files = []
    passed_keys = set([])
    for key, input_val in ((k, v) for (k, v) in inputs.items() if not k.startswith(("sentinel", "ignore"))):
        if key.endswith("_toolinput"):
            key = key.replace("_toolinput", "")
        if input_order[key] == "record":
            cur_keys, items = _read_cwl_record(input_val)
            passed_keys |= cur_keys
            items_by_key[key] = items
        else:
            items_by_key[tuple(key.split("__"))] = _cwlvar_to_wdl(input_val)
        input_files = _find_input_files(input_val, input_files)
    prepped = _merge_cwlinputs(items_by_key, input_order, parallel)
    out = []
    for data in prepped:
        if isinstance(data, (list, tuple)):
            out.append([_finalize_cwl_in(utils.to_single_data(x), work_dir, list(passed_keys),
                                         output_cwl_keys, runtime) for x in data])
        else:
            out.append(_finalize_cwl_in(data, work_dir, list(passed_keys), output_cwl_keys, runtime))
    return out, input_files
511409
def read_json(filepath, intkeys=True, intvalues=True):
    
    d = json.load(ensure_open(find_filepath(filepath), mode='rt'))
    d = update_dict_types(d, update_keys=intkeys, update_values=intvalues)
    return d
241349
def _dump(file_obj, options, out=sys.stdout):
    """"""
    # writer and keys are lazily loaded. We don't know the keys until we have
    # the first item. And we need the keys for the csv writer.
    total_count = 0
    writer = None
    keys = None
    for row in DictReader(file_obj, options.col):
        if not keys:
            keys = row.keys()
        if not writer:
            writer = csv.DictWriter(out, keys, delimiter=u'\t', quotechar=u'\'', quoting=csv.QUOTE_MINIMAL) \
                if options.format == 'csv' \
                else JsonWriter(out) if options.format == 'json' \
                else None
        if total_count == 0 and options.format == "csv" and not options.no_headers:
            writer.writeheader()
        if options.limit != -1 and total_count >= options.limit:
            return
        row_unicode = {k: v.decode("utf-8") if isinstance(v, bytes) else v for k, v in row.items()}
        writer.writerow(row_unicode)
        total_count += 1
122084
def execute(self, input_data):
        '''  '''
        raw_bytes = input_data['sample']['raw_bytes']
        strings = self.find_strings.findall(raw_bytes)
        return {'string_list': strings}
659506
def read_array(fname, sep=','):
    
    """
    with open(fname) as f:
        records = []
        for line in f:
            row = line.split(sep)
            record = [list(map(float, col.split())) for col in row]
            records.append(record)
        return numpy.array(records)
1097711
def read_records(file):
    """"""
    records = []
    for record_data in read_recordio(file):
        record = Record()
        record.ParseFromString(record_data)
        records.append(record)
    return records
444362
def _record_from_json(value, field):
    """"""
    if _not_null(value, field):
        record = {}
        record_iter = zip(field.fields, value["f"])
        for subfield, cell in record_iter:
            converter = _CELLDATA_FROM_JSON[subfield.field_type]
            if subfield.mode == "REPEATED":
                value = [converter(item["v"], subfield) for item in cell["v"]]
            else:
                value = converter(cell["v"], subfield)
            record[subfield.name] = value
        return record
552253
def rst_to_json(text):
    
    records = []
    last_type = None
    key = None
    data = {}
    directive = False

    lines = text.splitlines()
    for index, line in enumerate(lines):

        # check for directives
        if len(line) and line.strip().startswith(".."):
            directive = True
            continue

        # set the title
        if len(line) and (line[0] in string.ascii_letters or line[0].isdigit()):
            directive = False
            try:
                if lines[index + 1][0] not in DIVIDERS:
                    continue
            except IndexError:
                continue
            data = text_cleanup(data, key, last_type)
            data = {"title": line.strip()}
            records.append(
                data
            )
            continue

        # Grab standard fields (int, string, float)
        if len(line) and line[0].startswith(":"):
            data = text_cleanup(data, key, last_type)
            index = line.index(":", 1)
            key = line[1:index]
            value = line[index + 1:].strip()
            data[key], last_type = type_converter(value)
            directive = False
            continue

        # Work on multi-line strings
        if len(line) and line[0].startswith(" ") and directive == False:
            if not isinstance(data[key], str):
                # Not a string so continue on
                continue
            value = line.strip()
            if not len(value):
                # empty string, continue on
                continue
            # add next line
            data[key] += "\n{}".format(value)
            continue

        if last_type == STRING_TYPE and not len(line):
            if key in data.keys():
                data[key] += "\n"

    return json.dumps(records)
702418
def dump(self, file, sort_keys: bool = True, **kwargs) -> None:
        """"""
        json.dump(self.to_json(), file, sort_keys=sort_keys, **kwargs)
966349
def _serialize_inputs(inputs):
        """"""
        serialized_inputs = {}
        for input_id, input_value in inputs.items():
            if isinstance(input_value, list):
                serialized_list = Task._serialize_input_list(input_value)
                serialized_inputs[input_id] = serialized_list
            else:
                if isinstance(input_value, File):
                    input_value = Task._to_api_file_format(input_value)
                serialized_inputs[input_id] = input_value
        return serialized_inputs
933185
def main(args=None):

    # parse command-line options
    parser = argparse.ArgumentParser(prog='rect_wpoly_for_mos')
    # required arguments
    parser.add_argument("input_list",
                        help="TXT file with list JSON files derived from "
                             "longslit data")
    parser.add_argument("--fitted_bound_param", required=True,
                        help="Input JSON with fitted boundary parameters",
                        type=argparse.FileType('rt'))
    parser.add_argument("--out_MOSlibrary", required=True,
                        help="Output JSON file with results",
                        type=lambda x: arg_file_is_new(parser, x))
    # optional arguments
    parser.add_argument("--debugplot",
                        help="Integer indicating plotting & debugging options"
                             " (default=0)",
                        default=0, type=int,
                        choices=DEBUGPLOT_CODES)
    parser.add_argument("--echo",
                        help="Display full command line",
                        action="store_true")
    args = parser.parse_args(args)

    if args.echo:
        print('\033[1m\033[31m% ' + ' '.join(sys.argv) + '\033[0m\n')

    # ---

    # Read input TXT file with list of JSON files
    list_json_files = list_fileinfo_from_txt(args.input_list)
    nfiles = len(list_json_files)
    if abs(args.debugplot) >= 10:
        print('>>> Number of input JSON files:', nfiles)
        for item in list_json_files:
            print(item)
    if nfiles < 2:
        raise ValueError("Insufficient number of input JSON files")

    # read fitted boundary parameters and check that all the longslit JSON
    # files have been computed using the same fitted boundary parameters
    refined_boundary_model = RefinedBoundaryModelParam._datatype_load(
        args.fitted_bound_param.name)
    for ifile in range(nfiles):
        coef_rect_wpoly = RectWaveCoeff._datatype_load(
            list_json_files[ifile].filename)
        uuid_tmp = coef_rect_wpoly.meta_info['origin']['bound_param']
        if uuid_tmp[4:] != refined_boundary_model.uuid:
            print('Expected uuid:', refined_boundary_model.uuid)
            print('uuid for ifile #' + str(ifile + 1) + ": " + uuid_tmp)
            raise ValueError("Fitted boundary parameter uuid's do not match")

    # check consistency of grism, filter, DTU configuration and list of
    # valid slitlets
    coef_rect_wpoly_first_longslit = RectWaveCoeff._datatype_load(
        list_json_files[0].filename)
    filter_name = coef_rect_wpoly_first_longslit.tags['filter']
    grism_name = coef_rect_wpoly_first_longslit.tags['grism']
    dtu_conf = DtuConfiguration.define_from_dictionary(
        coef_rect_wpoly_first_longslit.meta_info['dtu_configuration']
    )
    list_valid_islitlets = list(range(1, EMIR_NBARS + 1))
    for idel in coef_rect_wpoly_first_longslit.missing_slitlets:
        list_valid_islitlets.remove(idel)
    for ifile in range(1, nfiles):
        coef_rect_wpoly = RectWaveCoeff._datatype_load(
            list_json_files[ifile].filename)
        filter_tmp = coef_rect_wpoly.tags['filter']
        if filter_name != filter_tmp:
            print(filter_name)
            print(filter_tmp)
            raise ValueError("Unexpected different filter found")
        grism_tmp = coef_rect_wpoly.tags['grism']
        if grism_name != grism_tmp:
            print(grism_name)
            print(grism_tmp)
            raise ValueError("Unexpected different grism found")
        coef_rect_wpoly = RectWaveCoeff._datatype_load(
            list_json_files[ifile].filename)
        dtu_conf_tmp = DtuConfiguration.define_from_dictionary(
            coef_rect_wpoly.meta_info['dtu_configuration']
        )
        if dtu_conf != dtu_conf_tmp:
            print(dtu_conf)
            print(dtu_conf_tmp)
            raise ValueError("Unexpected different DTU configurations found")
        list_valid_islitlets_tmp = list(range(1, EMIR_NBARS + 1))
        for idel in coef_rect_wpoly.missing_slitlets:
            list_valid_islitlets_tmp.remove(idel)
        if list_valid_islitlets != list_valid_islitlets_tmp:
            print(list_valid_islitlets)
            print(list_valid_islitlets_tmp)
            raise ValueError("Unexpected different list of valid slitlets")

    # check consistency of horizontal bounding box limits (bb_nc1_orig and
    # bb_nc2_orig) and ymargin_bb, and store the values for each slitlet
    dict_bb_param = {}
    print("Checking horizontal bounding box limits and ymargin_bb:")
    for islitlet in list_valid_islitlets:
        islitlet_progress(islitlet, EMIR_NBARS)
        cslitlet = 'slitlet' + str(islitlet).zfill(2)
        dict_bb_param[cslitlet] = {}
        for par in ['bb_nc1_orig', 'bb_nc2_orig', 'ymargin_bb']:
            value_initial = \
                coef_rect_wpoly_first_longslit.contents[islitlet - 1][par]
            for ifile in range(1, nfiles):
                coef_rect_wpoly = RectWaveCoeff._datatype_load(
                    list_json_files[ifile].filename)
                value_tmp = coef_rect_wpoly.contents[islitlet - 1][par]
                if value_initial != value_tmp:
                    print(islitlet, value_initial, value_tmp)
                    print(value_tmp)
                    raise ValueError("Unexpected different " + par)
                dict_bb_param[cslitlet][par] = value_initial
    print('OK!')

    # ---

    # Read and store all the longslit data
    list_coef_rect_wpoly = []
    for ifile in range(nfiles):
        coef_rect_wpoly = RectWaveCoeff._datatype_load(
            list_json_files[ifile].filename)
        list_coef_rect_wpoly.append(coef_rect_wpoly)

    # ---

    # Initialize structure to save results into an ouptut JSON file
    outdict = {}
    outdict['refined_boundary_model'] = refined_boundary_model.__getstate__()
    outdict['instrument'] = 'EMIR'
    outdict['meta_info'] = {}
    outdict['meta_info']['creation_date'] = datetime.now().isoformat()
    outdict['meta_info']['description'] = \
        'rectification and wavelength calibration polynomial coefficients ' \
        'as a function of csu_bar_slit_center for MOS'
    outdict['meta_info']['recipe_name'] = 'undefined'
    outdict['meta_info']['origin'] = {}
    outdict['meta_info']['origin']['wpoly_longslits'] = {}
    for ifile in range(nfiles):
        cdum = 'longslit_' + str(ifile + 1).zfill(3) + '_uuid'
        outdict['meta_info']['origin']['wpoly_longslits'][cdum] = \
            list_coef_rect_wpoly[ifile].uuid
    outdict['tags'] = {}
    outdict['tags']['grism'] = grism_name
    outdict['tags']['filter'] = filter_name
    outdict['dtu_configuration'] = dtu_conf.outdict()
    outdict['uuid'] = str(uuid4())
    outdict['contents'] = {}

    # include bb_nc1_orig, bb_nc2_orig and ymargin_bb for each slitlet
    # (note that the values of bb_ns1_orig and bb_ns2_orig cannot be
    # computed at this stage because they depend on csu_bar_slit_center)
    for islitlet in list_valid_islitlets:
        cslitlet = 'slitlet' + str(islitlet).zfill(2)
        outdict['contents'][cslitlet] = dict_bb_param[cslitlet]

    # check that order for rectification transformations is the same for all
    # the slitlets and longslit configurations
    order_check_list = []
    for ifile in range(nfiles):
        tmpdict = list_coef_rect_wpoly[ifile].contents
        for islitlet in list_valid_islitlets:
            ttd_order = tmpdict[islitlet - 1]['ttd_order']
            if ttd_order is not None:
                order_check_list.append(ttd_order)
            ttd_order_modeled = \
                tmpdict[islitlet - 1]['ttd_order_longslit_model']
            order_check_list.append(ttd_order_modeled)
    # remove duplicates in list
    order_no_duplicates = list(set(order_check_list))
    if len(order_no_duplicates) != 1:
        print('order_no_duplicates:', order_no_duplicates)
        raise ValueError('tdd_order is not constant!')
    ttd_order = int(order_no_duplicates[0])
    ncoef_rect = ncoef_fmap(ttd_order)
    if abs(args.debugplot) >= 10:
        print('>>> ttd_order........:', ttd_order)
        print('>>> ncoef_rect.......:', ncoef_rect)

    # check that polynomial degree in frontiers and spectrails are the same
    poldeg_check_list = []
    for ifile in range(nfiles):
        tmpdict = list_coef_rect_wpoly[ifile].contents
        for islitlet in list_valid_islitlets:
            tmppoly = tmpdict[islitlet - 1]['frontier']['poly_coef_lower']
            poldeg_check_list.append(len(tmppoly) - 1)
            tmppoly = tmpdict[islitlet - 1]['frontier']['poly_coef_upper']
            poldeg_check_list.append(len(tmppoly) - 1)
            tmppoly = tmpdict[islitlet - 1]['spectrail']['poly_coef_lower']
            poldeg_check_list.append(len(tmppoly) - 1)
            tmppoly = tmpdict[islitlet - 1]['spectrail']['poly_coef_middle']
            poldeg_check_list.append(len(tmppoly) - 1)
            tmppoly = tmpdict[islitlet - 1]['spectrail']['poly_coef_upper']
            poldeg_check_list.append(len(tmppoly) - 1)
    # remove duplicates in list
    poldeg_no_duplicates = list(set(poldeg_check_list))
    if len(poldeg_no_duplicates) != 1:
        print('poldeg_no_duplicates:', poldeg_no_duplicates)
        raise ValueError('poldeg is not constant in frontiers and '
                         'spectrails!')
    poldeg_spectrails = int(poldeg_no_duplicates[0])
    if abs(args.debugplot) >= 10:
        print('>>> poldeg spectrails:', poldeg_spectrails)

    # check that polynomial degree of wavelength calibration is the same for
    # all the slitlets
    poldeg_check_list = []
    for ifile in range(nfiles):
        tmpdict = list_coef_rect_wpoly[ifile].contents
        for islitlet in list_valid_islitlets:
            tmppoly = tmpdict[islitlet - 1]['wpoly_coeff']
            poldeg_check_list.append(len(tmppoly) - 1)
            tmppoly = tmpdict[islitlet - 1]['wpoly_coeff_longslit_model']
            poldeg_check_list.append(len(tmppoly) - 1)
    # remove duplicates in list
    poldeg_no_duplicates = list(set(poldeg_check_list))
    if len(poldeg_no_duplicates) != 1:
        print('poldeg_no_duplicates:', poldeg_no_duplicates)
        raise ValueError('poldeg is not constant in wavelength calibration '
                         'polynomials!')
    poldeg_wavecal = int(poldeg_no_duplicates[0])
    if abs(args.debugplot) >= 10:
        print('>>> poldeg wavecal...:', poldeg_wavecal)

    # ---

    # csu_bar_slit_center values for each slitlet
    print("CSU_bar_slit_center values:")
    for islitlet in list_valid_islitlets:
        islitlet_progress(islitlet, EMIR_NBARS)
        cslitlet = 'slitlet' + str(islitlet).zfill(2)
        list_csu_bar_slit_center = []
        for ifile in range(nfiles):
            tmpdict = list_coef_rect_wpoly[ifile].contents[islitlet - 1]
            csu_bar_slit_center = tmpdict['csu_bar_slit_center']
            list_csu_bar_slit_center.append(csu_bar_slit_center)
        # check that list_csu_bar_slit_center is properly sorted
        if not np.all(list_csu_bar_slit_center[:-1] <=
                  list_csu_bar_slit_center[1:]):
            print('cslitlet: ', cslitlet)
            print('list_csu_bar_slit_center: ', list_csu_bar_slit_center)
            raise ValueError('Unsorted list_csu_bar_slit_center')
        outdict['contents'][cslitlet]['list_csu_bar_slit_center'] = \
            list_csu_bar_slit_center
    print('OK!')

    # ---

    # rectification polynomial coefficients

    # note: when aij and bij have not been computed, we use the modeled
    # version aij_longslit_model and bij_longslit_model
    print("Rectification polynomial coefficients:")
    for islitlet in list_valid_islitlets:
        islitlet_progress(islitlet, EMIR_NBARS)
        cslitlet = 'slitlet' + str(islitlet).zfill(2)
        outdict['contents'][cslitlet]['ttd_order'] = ttd_order
        outdict['contents'][cslitlet]['ncoef_rect'] = ncoef_rect
        for keycoef in ['ttd_aij', 'ttd_bij', 'tti_aij', 'tti_bij']:
            for icoef in range(ncoef_rect):
                ccoef = str(icoef).zfill(2)
                list_cij = []
                for ifile in range(nfiles):
                    tmpdict = \
                        list_coef_rect_wpoly[ifile].contents[islitlet - 1]
                    cij = tmpdict[keycoef]
                    if cij is not None:
                        list_cij.append(cij[icoef])
                    else:
                        cij_modeled = tmpdict[keycoef + '_longslit_model']
                        if cij_modeled is None:
                            raise ValueError("Unexpected cij_modeled=None!")
                        else:
                            list_cij.append(cij_modeled[icoef])
                        if abs(args.debugplot) >= 10:
                            print("Warning: using " + keycoef +
                                  "_longslit_model for " + cslitlet +
                                  " in file " +
                                  list_json_files[ifile].filename)
                cdum = 'list_' + keycoef + '_' + ccoef
                outdict['contents'][cslitlet][cdum] = list_cij
    print('OK!')

    # ---

    # wavelength calibration polynomial coefficients

    # note: when wpoly_coeff have not been computed, we use the
    # wpoly_coeff_longslit_model
    print("Wavelength calibration polynomial coefficients:")
    for islitlet in list_valid_islitlets:
        islitlet_progress(islitlet, EMIR_NBARS)
        cslitlet = 'slitlet' + str(islitlet).zfill(2)
        outdict['contents'][cslitlet]['wpoly_degree'] = poldeg_wavecal
        for icoef in range(poldeg_wavecal + 1):
            ccoef = str(icoef).zfill(2)
            list_cij = []
            for ifile in range(nfiles):
                tmpdict = list_coef_rect_wpoly[ifile].contents[islitlet - 1]
                cij = tmpdict['wpoly_coeff']
                if cij is not None:
                    list_cij.append(cij[icoef])
                else:
                    cij_modeled = tmpdict['wpoly_coeff_longslit_model']
                    if cij_modeled is None:
                        raise ValueError("Unexpected cij_modeled=None!")
                    else:
                        list_cij.append(cij_modeled[icoef])
                    if abs(args.debugplot) >= 10:
                        print("Warning: using wpoly_coeff_longslit_model" +
                              " for " + cslitlet +
                              " in file " +
                              list_json_files[ifile].filename)
            outdict['contents'][cslitlet]['list_wpoly_coeff_' + ccoef] = \
                list_cij
    print('OK!')

    # ---

    # OBSOLETE
    # Save resulting JSON structure
    '''
    
    '''

    # --

    # Create object of type MasterRectWave with library of coefficients
    # for rectification and wavelength calibration
    master_rectwv = MasterRectWave(instrument='EMIR')
    master_rectwv.quality_control = numina.types.qc.QC.GOOD
    master_rectwv.tags['grism'] = grism_name
    master_rectwv.tags['filter'] = filter_name
    master_rectwv.meta_info['dtu_configuration'] = outdict['dtu_configuration']
    master_rectwv.meta_info['refined_boundary_model'] = {
        'parmodel': refined_boundary_model.meta_info['parmodel']
    }
    master_rectwv.meta_info['refined_boundary_model'].update(
        outdict['refined_boundary_model']['contents']
    )
    master_rectwv.total_slitlets = EMIR_NBARS
    master_rectwv.meta_info['origin'] = {
        'bound_param': 'uuid' + refined_boundary_model.uuid,
        'longslit_frames': ['uuid:' + list_coef_rect_wpoly[ifile].uuid
                            for ifile in range(nfiles)]
    }
    for i in range(EMIR_NBARS):
        islitlet = i + 1
        dumdict = {'islitlet': islitlet}
        cslitlet = 'slitlet' + str(islitlet).zfill(2)
        if cslitlet in outdict['contents']:
            dumdict.update(outdict['contents'][cslitlet])
        else:
            dumdict.update({
                'bb_nc1_orig': 0,
                'bb_nc2_orig': 0,
                'ymargin_bb': 0,
                'list_csu_bar_slit_center': [],
                'ttd_order': 0,
                'ncoef_rect': 0,
                'wpolydegree': 0
            })
            master_rectwv.missing_slitlets.append(islitlet)
        master_rectwv.contents.append(dumdict)
    master_rectwv.writeto(args.out_MOSlibrary.name)
    print('>>> Saving file ' + args.out_MOSlibrary.name)
642973
def read_json_breakdown(cls, fname):
        """"""
        if not os.path.exists(fname):
            raise RuntimeError

        with open(fname, 'r') as data_file:
            return cls.fixup_from_json(data_file.read())
729904
def scrub(data):
    """"""
    # blanks, Nones, and empty strings can stay as is
    if not data:
        return data
    if isinstance(data, (int, float)):
        return data
    if isinstance(data, list):
        return [scrub(entry) for entry in data]
    if isinstance(data, dict):
        return {scrub(key): scrub(value) for key, value in data.items()}
    try:
        return json.encoder.encode_basestring(data)[1:-1]
    except TypeError as exc:
        raise ValidationError("Input '%s' is not a permitted type: %s" % (data,
                                                                          exc))
    except Exception as exc:
        raise ValidationError("Input '%s' not permitted: %s" % (data, exc))
938333
def parse(self):
        """"""

        self.parse_fields()
        records = []
        for line in self.t['data'].split('\n'):
            if EMPTY_ROW.match(line):
                continue
            row = [self.autoconvert(line[start_field:end_field+1])
                   for start_field, end_field in self.fields]
            records.append(tuple(row))
        self.records = records
249364
def dumps(data, **kwargs):
    """"""

    def _encoder(value):
        if isinstance(value, datetime.datetime):
            return value.isoformat()

        if hasattr(value, "_data"):
            return value._data

        raise TypeError('Could not encode %r' % value)

    return json.dumps(data, default=_encoder, **kwargs)
766595
def read_input_data(filename):
    """"""
    logging.info('Opening file %s for reading input', filename)
    input_file = open(filename, 'r')
    data = []
    labels = []
    for line in input_file:
        tokens = line.split(',', 1)
        labels.append(tokens[0].strip())
        data.append(tokens[1].strip())
    return labels, data
608332
function convertAbcToJson(input) {
  var outputData = {};
  var parser = new Parser();
  parser.parse(input);
  var tune = parser.getTune();
  outputData = createJsonFromLines(tune);
  outputData.id = getJsonId(input);

  return JSON.stringify(outputData);
}