Creates a METADATA.json file from a data directory

    The file is written to output_path
4559
def create_metadata_file(output_path, data_dir):
    '''
    '''

    # Relative path to all (BASIS).metadata.json files
    meta_filelist, table_filelist, _, _ = get_all_filelist(data_dir)

    metadata = {}
    for meta_file_relpath in meta_filelist:

        # Read in the metadata for a single basis set
        meta_file_path = os.path.join(data_dir, meta_file_relpath)
        bs_metadata = read_json_basis(meta_file_path)

        # Base of the filename for table basis sets
        # Basename is something like '6-31G.', including the last period
        base_relpath, meta_filename = os.path.split(meta_file_relpath)
        base_filename = meta_filename.split('.')[0] + '.'

        # All the table files that correspond to this metadata file
        # (relative to data_dir)

        this_filelist = [
            x for x in table_filelist
            if os.path.dirname(x) == base_relpath and os.path.basename(x).startswith(base_filename)
        ]

        # The 'versions' dict that will go into the metadata
        version_info = {}

        # Make sure function types are the same
        function_types = None

        # For each table basis, compose it
        for table_file in this_filelist:
            # Obtain just the filename of the table basis
            table_filename = os.path.basename(table_file)

            # Obtain the base filename and version from the filename
            # The base filename is the part before the first period
            # (filebase.ver.table.json)
            table_filebase, ver, _, _ = table_filename.split('.')

            # Fully compose the basis set from components
            bs = compose_table_basis(table_file, data_dir)

            # Elements for which this basis is defined
            defined_elements = sorted(list(bs['elements'].keys()), key=lambda x: int(x))

            # Determine the types of functions contained in the basis
            # (gto, ecp, etc)
            if function_types is None:
                function_types = bs['function_types']
            elif function_types != bs['function_types']:
                raise RuntimeError("Differing function types across versions for " + base_filename)

            # Create the metadata for this specific version
            # yapf: disable
            version_info[ver] = { 'file_relpath': table_file,
                                  'revdesc': bs['revision_description'],
                                  'elements': defined_elements
                                }
            # yapf: enable

        # Sort the version dicts
        version_info = dict(sorted(version_info.items()))
        # Find the maximum version for this basis
        latest_ver = max(version_info.keys())

        # Create the common metadata for this basis set
        # display_name and other_names are placeholders to keep order
        # yapf: disable
        common_md = { 'display_name': None,
                      'other_names': None,
                      'description': bs['description'],
                      'latest_version': latest_ver,
                      'basename': base_filename[:-1], # Strip off that trailing period
                      'relpath': base_relpath,
                      'family': bs['family'],
                      'role': bs['role'],
                      'functiontypes': function_types,
                      'auxiliaries': bs['auxiliaries'],
                      'versions': version_info }
        # yapf: enable

        # Loop through all the common names, translate them, and then add the data
        for bs_name in bs_metadata['names']:
            tr_name = transform_basis_name(bs_name)

            if tr_name in metadata:
                raise RuntimeError("Duplicate basis set name: " + tr_name)

            # Create a new entry, with all the common metadata
            # Also, store the other names for this basis
            other_names = bs_metadata['names'].copy()
            other_names.remove(bs_name)
            metadata[tr_name] = common_md.copy()
            metadata[tr_name]['display_name'] = bs_name
            metadata[tr_name]['other_names'] = other_names

    # Write out the metadata
    metadata = dict(sorted(metadata.items()))
    _write_plain_json(output_path, metadata)
1288162
def generate_fileattr_metadata(local_path, metadata):
    # type: (blobxfer.models.upload.LocalPath, dict) -> dict
    
    if blobxfer.util.on_windows():
        global _FILEATTR_WARNED_ON_WINDOWS
        if not _FILEATTR_WARNED_ON_WINDOWS:
            _FILEATTR_WARNED_ON_WINDOWS = True
            logger.warning(
                'file attributes store/restore on Windows is not '
                'supported yet')
        return None
    else:
        md = {
            _JSON_KEY_FILE_ATTRIBUTES: {
                _JSON_KEY_FILE_ATTRIBUTES_POSIX: {
                    _JSON_KEY_FILE_ATTRIBUTES_MODE: local_path.mode,
                    _JSON_KEY_FILE_ATTRIBUTES_UID: local_path.uid,
                    _JSON_KEY_FILE_ATTRIBUTES_GID: local_path.gid,
                }
            }
        }
        return blobxfer.util.merge_dict(metadata, md)
1267419
def write_to_directory(self, dataset_info_dir):
    """"""
    # Save the metadata from the features (vocabulary, labels,...)
    if self.features:
      self.features.save_metadata(dataset_info_dir)

    if self.redistribution_info.license:
      with tf.io.gfile.GFile(self._license_filename(dataset_info_dir),
                             "w") as f:
        f.write(self.redistribution_info.license)

    with tf.io.gfile.GFile(self._dataset_info_filename(dataset_info_dir),
                           "w") as f:
      f.write(self.as_json)
792191
def _generate_metadata(path, cookbook_path, name):
    """"""
    global knife_installed
    if not knife_installed:
        return
    metadata_path_rb = os.path.join(path, 'metadata.rb')
    metadata_path_json = os.path.join(path, 'metadata.json')
    if (os.path.exists(metadata_path_rb) and
            (not os.path.exists(metadata_path_json) or
             os.stat(metadata_path_rb).st_mtime >
             os.stat(metadata_path_json).st_mtime)):
        error_msg = "Warning: metadata.json for {0}".format(name)
        error_msg += " in {0} is older that metadata.rb".format(cookbook_path)
        error_msg += ", cookbook attributes could be out of date\n\n"
        try:
            proc = subprocess.Popen(
                ['knife', 'cookbook', 'metadata', '-o', cookbook_path, name],
                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            resp, error = proc.communicate()
            if ('ERROR:' in resp or 'FATAL:' in resp
                    or 'Generating metadata for' not in resp):
                if("No user specified, pass via -u or specifiy 'node_name'"
                        in error):
                    error_msg += "You need to have an up-to-date (>=0.10.x)"
                    error_msg += " version of knife installed locally in order"
                    error_msg += " to generate metadata.json.\nError "
                else:
                    error_msg += "Unkown error "
                error_msg += "while executing knife to generate "
                error_msg += "metadata.json for {0}".format(path)
                print(error_msg)
                print resp
            if env.loglevel == 'debug':
                print "\n".join(resp.split("\n")[:2])
        except OSError:
            knife_installed = False
            error_msg += "If you locally install Chef's knife tool, LittleChef"
            error_msg += " will regenerate metadata.json files automatically\n"
            print(error_msg)
        else:
            print("Generated metadata.json for {0}\n".format(path))
144290
def save_metadata(self, phase, data_name):
        
        with open("{}/.metadata".format(make_path(phase)), "w+") as f:
            f.write("pipeline={}\nphase={}\ndata={}".format(self.pipeline_name, phase.phase_name,
                                                            data_name))
501582
def json_to_file(data, filename, pretty=False):
    ''''''
    kwargs = dict(indent=4) if pretty else {}
    dirname = os.path.dirname(filename)
    if not os.path.exists(dirname):
        os.makedirs(dirname)
    dump = json.dumps(api.__schema__, **kwargs)
    with open(filename, 'wb') as f:
        f.write(dump.encode('utf-8'))
662612
def do_copy_out(path_map)
      path_map.each do |src_path, dst_path|
        # constraining the generated 'metadata.json' size is debatable, but
        # our UI attempts to load metadata JSON into memory far too often to
        # be blasé about generating multi-megabyte JSON files.
        unless ::File.file?(src_path)
          raise MetadataError, "Expected generated file was not found: #{src_path}"
        end
        src_size = ::File.stat(src_path).size
        if src_size <= FREED_FILE_SIZE_CONSTRAINT
          ::FileUtils.mkdir_p(::File.dirname(dst_path))
          ::FileUtils.cp(src_path, dst_path)
        else
          raise MetadataError,
                "Generated file size of" +
                " #{src_size / 1024} KB" +
                " exceeded the allowed limit of" +
                " #{FREED_FILE_SIZE_CONSTRAINT / 1024} KB"
        end
      end
      true
    end
410804
def generate_json_artifacts(app, pagename, templatename, context, doctree):
    
    try:
        # We need to get the output directory where the docs are built
        # _build/json.
        build_json = os.path.abspath(
            os.path.join(app.outdir, '..', 'json')
        )
        outjson = os.path.join(build_json, pagename + '.fjson')
        outdir = os.path.dirname(outjson)
        if not os.path.exists(outdir):
            os.makedirs(outdir)
        with open(outjson, 'w+') as json_file:
            to_context = {
                key: context.get(key, '')
                for key in KEYS
            }
            json.dump(to_context, json_file, indent=4)
    except TypeError:
        log.exception(
            'Fail to encode JSON for page {page}'.format(page=outjson)
        )
    except IOError:
        log.exception(
            'Fail to save JSON output for page {page}'.format(page=outjson)
        )
    except Exception as e:
        log.exception(
            'Failure in JSON search dump for page {page}'.format(page=outjson)
        )
972299
def _create_data_files_directory(symlink=False):
        """"""
        current_directory = os.path.abspath(os.path.dirname(__file__))

        etc_kytos = os.path.join(BASE_ENV, ETC_KYTOS)

        if not os.path.exists(etc_kytos):
            os.makedirs(etc_kytos)

        src = os.path.join(current_directory, KYTOS_SKEL_PATH)
        dst = os.path.join(BASE_ENV, KYTOS_SKEL_PATH)

        if os.path.exists(dst):
            if not os.listdir(dst):
                # Path already exists but it's empty, so we'll populate it
                # We remove it first to avoid an exception from copytree
                os.rmdir(dst)
                shutil.copytree(src, dst)
        else:
            # It doesn't exist yet, so we should symlink or copy contents
            if symlink:
                os.symlink(src, dst)
            else:
                shutil.copytree(src, dst)
529636
def dump_metadata(filename, show_row_group_metadata, out=sys.stdout):
    
    def println(value):
        """Write a new line containing `value` to `out`."""
        out.write(value + "\n")
    footer = read_footer(filename)
    println("File Metadata: {0}".format(filename))
    println("  Version: {0}".format(footer.version))
    println("  Num Rows: {0}".format(footer.num_rows))
    println("  k/v metadata: ")
    if footer.key_value_metadata:
        for item in footer.key_value_metadata:
            println("    {0}={1}".format(item.key, item.value))
    else:
        println("    (none)")
    println("  schema: ")
    for element in footer.schema:
        println("    {name} ({type}): length={type_length}, "
                "repetition={repetition_type}, "
                "children={num_children}, "
                "converted_type={converted_type}".format(
                    name=element.name,
                    type=parquet_thrift.Type._VALUES_TO_NAMES[element.type]  # pylint: disable=protected-access
                    if element.type else None,
                    type_length=element.type_length,
                    repetition_type=_get_name(parquet_thrift.FieldRepetitionType,
                                              element.repetition_type),
                    num_children=element.num_children,
                    converted_type=element.converted_type))
    if show_row_group_metadata:
        println("  row groups: ")
        for row_group in footer.row_groups:
            num_rows = row_group.num_rows
            size_bytes = row_group.total_byte_size
            println(
                "  rows={num_rows}, bytes={bytes}".format(num_rows=num_rows,
                                                          bytes=size_bytes))
            println("    chunks:")
            for col_group in row_group.columns:
                cmd = col_group.meta_data
                println("      type={type} file_offset={offset} "
                        "compression={codec} "
                        "encodings={encodings} path_in_schema={path_in_schema} "
                        "num_values={num_values} uncompressed_bytes={raw_bytes} "
                        "compressed_bytes={compressed_bytes} "
                        "data_page_offset={data_page_offset} "
                        "dictionary_page_offset={dictionary_page_offset}".format(
                            type=_get_name(parquet_thrift.Type, cmd.type),
                            offset=col_group.file_offset,
                            codec=_get_name(parquet_thrift.CompressionCodec, cmd.codec),
                            encodings=",".join(
                                [_get_name(
                                    parquet_thrift.Encoding, s) for s in cmd.encodings]),
                            path_in_schema=cmd.path_in_schema,
                            num_values=cmd.num_values,
                            raw_bytes=cmd.total_uncompressed_size,
                            compressed_bytes=cmd.total_compressed_size,
                            data_page_offset=cmd.data_page_offset,
                            dictionary_page_offset=cmd.dictionary_page_offset))
                with open(filename, 'rb') as file_obj:
                    offset = _get_offset(cmd)
                    file_obj.seek(offset, 0)
                    values_read = 0
                    println("      pages: ")
                    while values_read < num_rows:
                        page_header = _read_page_header(file_obj)
                        # seek past current page.
                        file_obj.seek(page_header.compressed_page_size, 1)
                        daph = page_header.data_page_header
                        type_ = _get_name(parquet_thrift.PageType, page_header.type)
                        raw_bytes = page_header.uncompressed_page_size
                        num_values = None
                        if page_header.type == parquet_thrift.PageType.DATA_PAGE:
                            num_values = daph.num_values
                            values_read += num_values
                        if page_header.type == parquet_thrift.PageType.DICTIONARY_PAGE:
                            pass

                        encoding_type = None
                        def_level_encoding = None
                        rep_level_encoding = None
                        if daph:
                            encoding_type = _get_name(parquet_thrift.Encoding, daph.encoding)
                            def_level_encoding = _get_name(
                                parquet_thrift.Encoding, daph.definition_level_encoding)
                            rep_level_encoding = _get_name(
                                parquet_thrift.Encoding, daph.repetition_level_encoding)

                        println("        page header: type={type} "
                                "uncompressed_size={raw_bytes} "
                                "num_values={num_values} encoding={encoding} "
                                "def_level_encoding={def_level_encoding} "
                                "rep_level_encoding={rep_level_encoding}".format(
                                    type=type_,
                                    raw_bytes=raw_bytes,
                                    num_values=num_values,
                                    encoding=encoding_type,
                                    def_level_encoding=def_level_encoding,
                                    rep_level_encoding=rep_level_encoding))
135370
def write_config_json(config_file, data):
    
    outfile = None
    try:
        with open(config_file, 'w') as outfile:
            json.dump(data, outfile)
    except:
        line, filename, synerror = trace()
        raise ArcRestHelperError({
                    "function": "init_config_json",
                    "line": line,
                    "filename":  filename,
                    "synerror": synerror,
                                    }
                                    )
    finally:
        outfile = None

        del outfile

        gc.collect()
299662
def _save_uploaded_data_json(samples, data_json_work, out_dir):
    
    if not utils.file_exists(data_json_work):
        return None

    upload_path_mapping = dict()
    for sample in samples:
        upload_path_mapping.update(get_all_upload_paths_from_sample(sample))
    if not upload_path_mapping:
        return data_json_work

    with io.open(data_json_work, encoding="utf-8") as f:
        data = json.load(f, object_pairs_hook=OrderedDict)
    upload_base = samples[0]["upload"]["dir"]
    data = walk_json(data, lambda s: _work_path_to_rel_final_path(s, upload_path_mapping, upload_base))

    data_json_final = os.path.join(out_dir, "multiqc_data_final.json")
    with io.open(data_json_final, "w", encoding="utf-8") as f:
        json.dump(data, f, indent=4)
    return data_json_final
1361358
def metadata_index_json(self):
        """"""
        try:
            return op.join(self.metadata_dir, 'INDEX.json')
        except FileNotFoundError:
            return op.join(self.metadata_dir, 'INDEX_JSON')
1224064
def atomic_create_file(file_path):
    """"""
    fd, tmp_file_path = tempfile.mkstemp(
        prefix=os.path.basename(file_path), dir=os.path.dirname(file_path), suffix=".metadata_tmp"
    )
    try:
        with os.fdopen(fd, "w") as out_file:
            yield out_file

        os.rename(tmp_file_path, file_path)
    except Exception:  # pytest: disable=broad-except
        with contextlib.suppress(Exception):
            os.unlink(tmp_file_path)
        raise
705898
def upload_metadata_cli(directory, create_csv='', review='',
                        max_size='128m', verbose=False, debug=False):
    
    return upload_metadata(directory, create_csv, review,
                           max_size, verbose, debug)
1283778
function (output) {
              // Manually convert paths to relative, because LESS
              // can make tose absolute for complex cases.
              var map = JSON.parse(output);
              var dir = path.dirname(context.pathname);
              map.sources.forEach(function (file, idx) {
                var rel = path.relative(dir, file);
                if (path.sep === '\\') { rel = rel.replace('\\', '/'); }
                map.sources[idx] = rel;
              });

              self.map = JSON.stringify(map);
            }
1125055
def save_local_metadata(self):
        
        # save to the set local path and add .json
        with open(self.filepath + '.json', 'w+') as outfile:
            json.dump(self._prep_metadata(), outfile, indent=4)