takes the referenced files and loads them
    returns the updated schema
    :param the_list:
    :param val:
    :param a_scheme:
    :param a_property:
    :return:
353347
def _load_referenced_schemes_from_list(the_list, val, a_scheme, a_property):
    
    scheme = copy.copy(a_scheme)
    new_list = []
    if isinstance(the_list, list):
        for an_item in the_list:
            if ((not isinstance(an_item, basestring)) and
                    (u'$ref' in an_item.keys())):
                sub_scheme_name = generate_schema_name_from_uri(an_item['$ref'])
                content = load_schema(sub_scheme_name)
                new_list.append(content)
    else:
        # somewhere the array is not an array - payment_reminder
        sub_scheme_name = generate_schema_name_from_uri(the_list['$ref'])
        new_list = load_schema(sub_scheme_name)
    scheme['properties'][a_property]['items'] = new_list
    return scheme
1045184
def schema_resolve_refs(schema, ref_resolver=None, root=None):
    
    # FIXME more stable implementation that only attempts to resolve {"$ref"} objects where they are allowed.
    if isinstance(schema, dict):
        if len(schema) == 1 and "$ref" in schema and isinstance(schema["$ref"], six.string_types):
            reference = schema["$ref"]
            if reference.startswith("#"):
                # TODO should also resolve any paths within the reference, which would need to be deferred.
                return root
            return ref_resolver(reference)

        resolved = {}
        for k, v in schema.items():
            resolved[k] = schema_resolve_refs(v,
                                              ref_resolver=ref_resolver,
                                              root=root if root is not None else resolved)
        return resolved
    if isinstance(schema, (list, tuple)):
        return [schema_resolve_refs(v, ref_resolver=ref_resolver, root=root) for v in schema]
    return schema
866106
def load_manifest_file(client, bucket, schema, versioned, ifilters, key_info):
    
    # To avoid thundering herd downloads, we do an immediate yield for
    # interspersed i/o
    yield None

    # Inline these values to avoid the local var lookup, they are constants
    # rKey = schema['Key'] # 1
    # rIsLatest = schema['IsLatest'] # 3
    # rVersionId = schema['VersionId'] # 2

    with tempfile.NamedTemporaryFile() as fh:
        client.download_fileobj(Bucket=bucket, Key=key_info['key'], Fileobj=fh)
        fh.seek(0)
        reader = csv.reader(gzip.GzipFile(fileobj=fh, mode='r'))
        for key_set in chunks(reader, 1000):
            keys = []
            for kr in key_set:
                k = kr[1]
                if inventory_filter(ifilters, schema, kr):
                    continue
                k = unquote_plus(k)
                if versioned:
                    if kr[3] == 'true':
                        keys.append((k, kr[2], True))
                    else:
                        keys.append((k, kr[2]))
                else:
                    keys.append(k)
            yield keys
534154
function loadSchema(schemaBasePath, fullSchemaFileName) {
    var scfPath = fullSchemaFileName.split('/');
    var schemaFileName = scfPath[scfPath.length - 1];
    var mainSchema = null;

    // First, register() the main schemas you plan to use.
    try {
        mainSchema = _jsYaml2.default.load(_fs2.default.readFileSync(_path2.default.resolve(schemaBasePath, schemaFileName), 'utf-8'));
        var missingSchemas = js.register(mainSchema);

        // Next, load the missing sub-schemas recursively
        missingSchemas.forEach(function (missingSchema) {
            loadSchema(schemaBasePath, missingSchema);
        });
    } catch (err) {
        //console.log(err)
    }

    return mainSchema;
}
277492
function calcDependList(file) {
		calcing[file] = true;
		var reqs = file_req[file];
		if (reqs) {
			for (var i = 0, len = reqs.length, fi; i < len; i++) {
				fi = reqs[i];
				if (depended[fi] || calcing[fi]) {
					continue;
				}
				calcDependList(fi);
			}
		}
		all_depended[file] = depended[file] = true;

		var encoded_name = versions[file];
		if (encoded_name) {
			encoded_name = encoded_name[0];
		} else {
			encoded_name = last_file = addOne(last_file);
			versions[file] = [encoded_name, CURRENT_VER, '', file, 'new'];
		}
		depended_list.push(encoded_name);
	}
1223768
def resolving(self, ref: str):
        
        new_uri = urlparse.urljoin(self.resolution_scope, ref)
        uri, fragment = urlparse.urldefrag(new_uri)

        if normalize(uri) in self.store:
            schema = self.store[normalize(uri)]
        elif not uri or uri == self.base_uri:
            schema = self.schema
        else:
            schema = resolve_remote(uri, self.handlers)
            if self.cache:
                self.store[normalize(uri)] = schema

        old_base_uri, old_schema = self.base_uri, self.schema
        self.base_uri, self.schema = uri, schema
        try:
            with self.in_scope(uri):
                yield resolve_path(schema, fragment)
        finally:
            self.base_uri, self.schema = old_base_uri, old_schema
642943
def schema_files():
    '''
    '''
    def recursive_glob(path, patterns):
        results = []
        for base, dirs, files in os.walk(path):
            matching_files = []
            for pattern in patterns:
                matching_files.extend(fnmatch.filter(files, pattern))
            results.extend(os.path.join(base, f) for f in matching_files)
        return results

    base_folder = 'datapackage_registry'
    remove_base_folder = lambda path: path[len(base_folder) + 1:]

    path = os.path.join(base_folder, 'schemas')
    files_paths = recursive_glob(path, ['*.csv', '*.json'])

    return [remove_base_folder(f) for f in files_paths]
357592
private List<String> getSchemaStringsFromDir(File dir) {
    List<String> schemaStrings = new ArrayList<String>();
    Collection<File> schemaFiles = FileUtils.listFiles(dir,
        new SuffixFileFilter(".avsc"), TrueFileFilter.INSTANCE);
    for (File schemaFile : schemaFiles) {
      schemaStrings.add(getSchemaStringFromFile(schemaFile));
    }
    return schemaStrings;
  }
1321085
def load_schemes(workspace_dir_path)
      # Normalizes path to directory of workspace needed for file_reference.absolute_path
      workspaces_dir = workspace_dir_path
      if File.extname(workspace_dir_path) == '.xcworkspace'
        workspaces_dir = File.expand_path('..', workspaces_dir)
      end

      file_references.each do |file_reference|
        project_full_path = file_reference.absolute_path(workspaces_dir)
        load_schemes_from_project(project_full_path)
      end

      # Load schemes that are in the workspace container.
      workspace_abs_path = File.absolute_path(workspace_dir_path)
      Dir[File.join(workspace_dir_path, 'xcshareddata', 'xcschemes', '*.xcscheme')].each do |scheme|
        scheme_name = File.basename(scheme, '.xcscheme')
        @schemes[scheme_name] = workspace_abs_path
      end
    end
232240
def _resolve_definitions(self, schema, definitions):
        
        if not definitions:
            return schema

        if not isinstance(schema, dict):
            return schema

        ref = schema.pop('$ref', None)
        if ref:
            path = ref.split('/')[2:]
            definition = definitions
            for component in path:
                definition = definitions[component]

            if definition:
                # Only update specified fields
                for (key, val) in six.iteritems(definition):
                    if key not in schema:
                        schema[key] = val

        for key in six.iterkeys(schema):
            schema[key] = self._resolve_definitions(schema[key], definitions)

        return schema
1273649
function CreateSchemedProperty(object, scheme, schema_name, index) {
    if (object[schema_name])
        return;

    Object.defineProperty(object, schema_name, {
        configurable: false,
        enumerable: true,
        get: function() {
            return this.getHook(schema_name, this.prop_array[index]);
        },
        set: function(value) {

            let result = { valid: false };

            let val = scheme.parse(value);

            scheme.verify(val, result);

            if (result.valid && this.prop_array[index] != val) {
                this.prop_array[index] = this.setHook(schema_name, val);
                this.scheduleUpdate(schema_name);
                this._changed_ = true;
            }
        }
    });
}
208438
def update_types(config, reference, list_sep=':'):
    

    def _coerce(current, value):
        # Coerce a value to the `current` type.
        try:
            # First we try to apply current to the value, since it
            # might be a function
            return current(value)
        except TypeError:
            # Then we check if current is a list AND if the value
            # is a string.
            if isinstance(current, list) and isinstance(value, str):
                # If so, we use the colon as the separator
                return value.split(list_sep)

            try:
                # If we are here, we should try to apply the type
                # of `current` to the value
                return type(current)(value)
            except TypeError:
                # Worst case scenario we return the value itself.
                return value

    def _update_type(value, path):
        current = reference

        for elem in path:
            try:
                current = current[elem]
            except KeyError:
                return value

        return _coerce(current, value)

    return map_leafs(_update_type, config)
410991
def update_input(filelist, ivmlist=None, removed_files=None):
    
    newfilelist = []

    if removed_files == []:
        return filelist, ivmlist
    else:
        sci_ivm = list(zip(filelist, ivmlist))
        for f in removed_files:
            result=[sci_ivm.remove(t) for t in sci_ivm if t[0] == f ]
        ivmlist = [el[1] for el in sci_ivm]
        newfilelist = [el[0] for el in sci_ivm]
        return newfilelist, ivmlist
620938
def load_custom_schema(self, type, schema_file_path=None):
        
        try:
            if type in self.custom_data_schemas:
                return self.custom_data_schemas[type]

            if schema_file_path:
                _schema_file = schema_file_path
            else:
                _schema_file = os.path.join(self.base_path, 'schemas', "{0}_schema.json".format(type))

            custom_data_schema = json.load(open(_schema_file, 'r'))
            self.custom_data_schemas[type] = custom_data_schema

            return custom_data_schema
        except Exception as e:
            raise UnsupportedDataSchemaException(
                message="There is no schema defined for the '{0}' data type.".format(type))
391250
def schema(ds):
    
    tables, ref_tables = {}, {}
    table_lookup = {t.url.string: t for t in ds.tables if ds.get_tabletype(t)}
    for table in table_lookup.values():
        spec = TableSpec(ds.get_tabletype(table))
        spec.primary_key = [
            c for c in table.tableSchema.columns if
            c.propertyUrl and c.propertyUrl.uri == term_uri('id')][0].name
        # Map the column name to the default:
        if spec.name in PROPERTY_URL_TO_COL:
            spec.primary_key = PROPERTY_URL_TO_COL[spec.name][term_uri('id')]
        for c in table.tableSchema.columns:
            if c.propertyUrl and c.propertyUrl.uri == term_uri('source'):
                # A column referencing sources is replaced by an association table.
                otype = ds.get_tabletype(table).replace('Table', '')
                ref_tables[ds.get_tabletype(table)] = TableSpec(
                    '{0}Source'.format(otype),  # The name of the association table.
                    [ColSpec(otype + '_ID'), ColSpec('Source_ID'), ColSpec('Context')],
                    [
                        (  # The foreign key to the referencing object:
                            ['dataset_ID', otype + '_ID'],
                            ds.get_tabletype(table),
                            ['dataset_ID', spec.primary_key]),
                        (  # The foreign key to the referenced source:
                            ['dataset_ID', 'Source_ID'],
                            'SourceTable',
                            ['dataset_ID', 'ID']),
                    ],
                    c.name)
            else:
                cname = c.header
                if c.propertyUrl and spec.name in PROPERTY_URL_TO_COL:
                    if c.propertyUrl.uri in PROPERTY_URL_TO_COL[spec.name]:
                        cname = PROPERTY_URL_TO_COL[spec.name][c.propertyUrl.uri]
                spec.columns.append(ColSpec(
                    cname,
                    c.datatype.base if c.datatype else c.datatype,
                    c.separator,
                    cname == spec.primary_key,
                    cldf_name=c.header))
        for fk in table.tableSchema.foreignKeys:
            if fk.reference.schemaReference:
                # We only support Foreign Key references between tables!
                continue  # pragma: no cover
            ref = table_lookup[fk.reference.resource.string]
            ref_type = ds.get_tabletype(ref)
            if ref_type:
                colRefs = sorted(fk.columnReference)
                if spec.name in PROPERTY_URL_TO_COL:
                    # Must map foreign keys
                    colRefs = []
                    for c in sorted(fk.columnReference):
                        c = ds[spec.name, c]
                        if c.propertyUrl and c.propertyUrl.uri in PROPERTY_URL_TO_COL[spec.name]:
                            colRefs.append(PROPERTY_URL_TO_COL[spec.name][c.propertyUrl.uri])
                        else:
                            colRefs.append(c.header)
                rcolRefs = sorted(fk.reference.columnReference)
                if ref_type in PROPERTY_URL_TO_COL:
                    # Must map foreign key targets!
                    rcolRefs = []
                    for c in sorted(fk.reference.columnReference):
                        c = ds[ref_type, c]
                        if c.propertyUrl and c.propertyUrl.uri in PROPERTY_URL_TO_COL[ref_type]:
                            rcolRefs.append(PROPERTY_URL_TO_COL[ref_type][c.propertyUrl.uri])
                        else:
                            rcolRefs.append(c.header)
                spec.foreign_keys.append((
                    tuple(['dataset_ID'] + colRefs),
                    ds.get_tabletype(table_lookup[fk.reference.resource.string]),
                    tuple(['dataset_ID'] + rcolRefs)))
        tables[spec.name] = spec

    # must determine the order in which tables must be created!
    ordered = OrderedDict()
    i = 0
    #
    # We loop through the tables repeatedly, and whenever we find one, which has all
    # referenced tables already in ordered, we move it from tables to ordered.
    #
    while tables and i < 100:
        i += 1
        for table in list(tables.keys()):
            if all(ref[1] in ordered for ref in tables[table].foreign_keys):
                # All referenced tables are already created.
                ordered[table] = tables.pop(table)
                break
    if tables:  # pragma: no cover
        raise ValueError('there seem to be cyclic dependencies between the tables')

    return list(ordered.values()), ref_tables
1031577
def file_list(ref)
      if (sha = @access.ref_to_sha(ref))
        commit = @access.commit(sha)
        tree_map_for(sha).inject([]) do |list, entry|
          next list if entry.name.start_with?('_')
          next list if @page_class.valid_page_name?(entry.name)
          list << entry.file(self, commit)
        end
      else
        []
      end
    end
1246274
def get_scheme_files(patterns=None):
    """"""
    patterns = patterns or ['*']
    pattern_list = ['{}.yaml'.format(pattern) for pattern in patterns]
    scheme_files = []
    for scheme_path in get_scheme_dirs():
        for pattern in pattern_list:
            file_paths = glob(os.path.join(scheme_path, pattern))
            scheme_files.extend(file_paths)

    return scheme_files