Uploads metadata individually by language to help identify exactly which items have issues
512723
def upload_individual_item(app, version, language, key, value)
      details = app.details
      version.send(key)[language] = value if LOCALISED_VERSION_VALUES.include?(key)
      details.send(key)[language] = value if LOCALISED_APP_VALUES.include?(key)
      Helper.show_loading_indicator("Uploading #{language} #{key} to App Store Connect")
      version.save!
      Helper.hide_loading_indicator
      begin
        details.save!
        UI.success("Successfully uploaded #{language} #{key} to App Store Connect")
      rescue Spaceship::TunesClient::ITunesConnectError => e
        UI.error("Error in #{language} #{key}: \n#{value}")
        UI.error(e.message) # Don't use user_error to allow all values to get checked
      end
    end
1315112
def _process_items(items, user_conf, error_protocol):
    
    def process_meta(item, error_protocol):
        try:
            return item._parse()
        except Exception, e:
            error_protocol.append(
                "Can't parse %s: %s" % (item._get_filenames()[0], e.message)
            )

            if isinstance(item, DataPair):
                return item.ebook_file

    # process all items and put them to output queue
    out = []
    for item in items:
        if isinstance(item, EbookFile):
            out.append(item)
        else:
            out.append(process_meta(item, error_protocol))
    out = filter(lambda x: x, out)  # remove None items (process_meta() fails)

    # remove processed files
    fn_pool = []
    soon_removed = out if conf_merger(user_conf, "LEAVE_BAD_FILES") else items
    for item in soon_removed:
        fn_pool.extend(item._get_filenames())

    _remove_files(fn_pool)

    return out
876451
def _find_bad_meta(self):
        ''''''
        self._badmeta = dict()

        for datatype in self.meta:
            for item in self.meta[datatype]:
                if not Dap._meta_valid[datatype].match(item):
                    if datatype not in self._badmeta:
                        self._badmeta[datatype] = []
                    self._badmeta[datatype].append(item)
705878
def _validate_metadata(metadata_props):
    '''
    
    '''
    if len(CaseInsensitiveDict(metadata_props)) != len(metadata_props):
        raise RuntimeError('Duplicate metadata props found')

    for key, value in metadata_props.items():
        valid_values = KNOWN_METADATA_PROPS.get(key)
        if valid_values and value.lower() not in valid_values:
            warnings.warn('Key {} has invalid value {}. Valid values are {}'.format(key, value, valid_values))
744720
def determine_metadata(self, request, view):
        """"""
        metadata = super(
            DynamicMetadata,
            self).determine_metadata(
            request,
            view)
        metadata['features'] = getattr(view, 'features', [])
        if hasattr(view, 'get_serializer'):
            serializer = view.get_serializer(dynamic=False)
            if hasattr(serializer, 'get_name'):
                metadata['resource_name'] = serializer.get_name()
            if hasattr(serializer, 'get_plural_name'):
                metadata['resource_name_plural'] = serializer.get_plural_name()
        metadata['properties'] = self.get_serializer_info(serializer)
        return metadata
740131
def _init_metadata(self):
        """"""
        ItemTextsFormRecord._init_metadata(self)
        ItemFilesFormRecord._init_metadata(self)
        edXBaseFormRecord._init_metadata(self)
        IRTItemFormRecord._init_metadata(self)
        TimeValueFormRecord._init_metadata(self)
        ProvenanceFormRecord._init_metadata(self)
        super(edXItemFormRecord, self)._init_metadata()
413311
function MetaData() {

    // the key for the storing
    this.key = null;
    // data to store
    this.value = null;
    // temporary filename for the cached file because filenames cannot represend urls completely
    this.filename = null;
    // expirydate of the entry
    this.expires = null;
    // size of the current entry
    this.size = null;
}
196339
def verify_available_languages!(options)
      return if options[:skip_metadata]

      # Collect all languages we need
      # We only care about languages from user provided values
      # as the other languages are on iTC already anyway
      v = options[:app].edit_version(platform: options[:platform])
      UI.user_error!("Could not find a version to edit for app '#{options[:app].name}', the app metadata is read-only currently") unless v

      enabled_languages = options[:languages] || []
      LOCALISED_VERSION_VALUES.each do |key|
        current = options[key]
        next unless current && current.kind_of?(Hash)
        current.each do |language, value|
          language = language.to_s
          enabled_languages << language unless enabled_languages.include?(language)
        end
      end

      # Reject "default" language from getting enabled
      # because "default" is not an iTC language
      enabled_languages = enabled_languages.reject do |lang|
        lang == "default"
      end.uniq

      if enabled_languages.count > 0
        v.create_languages(enabled_languages)
        lng_text = "language"
        lng_text += "s" if enabled_languages.count != 1
        Helper.show_loading_indicator("Activating #{lng_text} #{enabled_languages.join(', ')}...")
        v.save!
        Helper.hide_loading_indicator
      end
      true
    end
1066872
def upload(
    ctx, files, max_threads, prompt, forward, reverse, tags, metadata, project_id, coerce_ascii
):
    """"""

    appendables = {}
    if tags:
        appendables["tags"] = []
        for tag in tags:
            appendables["tags"].append(tag)

    if metadata:
        appendables["metadata"] = {}
        for metadata_kv in metadata:
            split_metadata = metadata_kv.split("=", 1)
            if len(split_metadata) > 1:
                metadata_value = split_metadata[1]
                appendables["metadata"][split_metadata[0]] = metadata_value

    appendables = validate_appendables(appendables, ctx.obj["API"])

    if (forward or reverse) and not (forward and reverse):
        click.echo("You must specify both forward and reverse files", err=True)
        ctx.exit(1)

    if forward and reverse:
        if len(files) > 0:
            click.echo(
                "You may not pass a FILES argument when using the "
                " --forward and --reverse options.",
                err=True,
            )
            ctx.exit(1)
        files = [(forward, reverse)]
    elif len(files) == 0:
        click.echo(ctx.get_help())
        return
    else:
        files = list(files)

        # "intelligently" find paired files and tuple them
        paired_files = []
        single_files = set(files)

        for filename in files:
            # convert "read 1" filenames into "read 2" and check that they exist; if they do
            # upload the files as a pair, autointerleaving them
            pair = re.sub("[._][Rr]1[._]", lambda x: x.group().replace("1", "2"), filename)

            # we don't necessary need the R2 to have been passed in; we infer it anyways
            if pair != filename and os.path.exists(pair):
                if not prompt and pair not in single_files:
                    # if we're not prompting, don't automatically pull in files
                    # not in the list the user passed in
                    continue

                paired_files.append((filename, pair))

                if pair in single_files:
                    single_files.remove(pair)

                single_files.remove(filename)

        auto_pair = True

        if prompt and len(paired_files) > 0:
            pair_list = ""
            for p in paired_files:
                pair_list += "\n  {}  &  {}".format(os.path.basename(p[0]), os.path.basename(p[1]))

            answer = click.confirm(
                "It appears there are paired files:{}\nInterleave them after upload?".format(
                    pair_list
                ),
                default="Y",
            )

            if not answer:
                auto_pair = False

        if auto_pair:
            files = paired_files + list(single_files)

    total_size = sum(
        [
            (_file_size(x[0], uncompressed=True) + _file_size(x[1], uncompressed=True))
            if isinstance(x, tuple)
            else _file_size(x, uncompressed=False)
            for x in files
        ]
    )

    upload_kwargs = {
        "metadata": appendables["valid_metadata"],
        "tags": appendables["valid_tags"],
        "project": project_id,
        "coerce_ascii": coerce_ascii,
        "progressbar": progressbar(length=total_size, label="Uploading..."),
    }

    run_via_threadpool(
        ctx.obj["API"].Samples.upload,
        files,
        upload_kwargs,
        max_threads=max_threads,
        graceful_exit=False,
    )
963133
private function _checkfiles()
    {
        if (!file_exists($this->_plugin_base."js/".strtolower($this->_plugin_name).".js")) {
            $this->global_error->addError("file_exists(".$this->_plugin_base."js/".strtolower($this->_plugin_name).".js)", "JS-File for Plugin '".$this->_plugin_name."' is missing!");
        } else {
            if (!is_readable($this->_plugin_base."js/".strtolower($this->_plugin_name).".js")) {
                $this->global_error->addError("is_readable(".$this->_plugin_base."js/".strtolower($this->_plugin_name).".js)", "JS-File for Plugin '".$this->_plugin_name."' is not readable but present!");
            }
        }
        if (!file_exists($this->_plugin_base."lang/en.xml")) {
            $this->global_error->addError("file_exists(".$this->_plugin_base."lang/en.xml)", "At least an english translation must exist for the plugin!");
        } else {
            if (!is_readable($this->_plugin_base."lang/en.xml")) {
                $this->global_error->addError("is_readable(".$this->_plugin_base."js/".$this->_plugin_name.".js)", "The english translation can't be read but is present!");
            }
        }
    }
808871
def _CheckData(yaml_data):
  """"""
  legal_keys = set(('blacklist', 'whitelist'))
  unknown_keys = set(yaml_data) - legal_keys
  if unknown_keys:
    raise UnknownConfigKeyError(
        'Unknown keys in configuration: %s' % unknown_keys)

  for key, data in six.iteritems(yaml_data):
    _AssertDataIsList(key, data)
353063
def check_info(info):
    
    if not isinstance(info, dict):
        raise ValueError("bad metainfo - not a dictionary")

    pieces = info.get("pieces")
    if not isinstance(pieces, basestring) or len(pieces) % 20 != 0:
        raise ValueError("bad metainfo - bad pieces key")

    piece_size = info.get("piece length")
    if not isinstance(piece_size, (int, long)) or piece_size <= 0:
        raise ValueError("bad metainfo - illegal piece length")

    name = info.get("name")
    if not isinstance(name, basestring):
        raise ValueError("bad metainfo - bad name (type is %r)" % type(name).__name__)
    if not ALLOWED_ROOT_NAME.match(name):
        raise ValueError("name %s disallowed for security reasons" % name)

    if ("files" in info) == ("length" in info):
        raise ValueError("single/multiple file mix")

    if "length" in info:
        length = info.get("length")
        if not isinstance(length, (int, long)) or length < 0:
            raise ValueError("bad metainfo - bad length")
    else:
        files = info.get("files")
        if not isinstance(files, (list, tuple)):
            raise ValueError("bad metainfo - bad file list")

        for item in files:
            if not isinstance(item, dict):
                raise ValueError("bad metainfo - bad file value")

            length = item.get("length")
            if not isinstance(length, (int, long)) or length < 0:
                raise ValueError("bad metainfo - bad length")

            path = item.get("path")
            if not isinstance(path, (list, tuple)) or not path:
                raise ValueError("bad metainfo - bad path")

            for part in path:
                if not isinstance(part, basestring):
                    raise ValueError("bad metainfo - bad path dir")
                part = fmt.to_unicode(part)
                if part == '..':
                    raise ValueError("relative path in %s disallowed for security reasons" % '/'.join(path))
                if part and not ALLOWED_PATH_NAME.match(part):
                    raise ValueError("path %s disallowed for security reasons" % part)

        file_paths = [os.sep.join(item["path"]) for item in files]
        if len(set(file_paths)) != len(file_paths):
            raise ValueError("bad metainfo - duplicate path")

    return info
498476
def clean_meta(meta, including_info=False, logger=None):
    
    modified = set()

    for key in meta.keys():
        if [key] not in METAFILE_STD_KEYS:
            if logger:
                logger("Removing key %r..." % (key,))
            del meta[key]
            modified.add(key)

    if including_info:
        for key in meta["info"].keys():
            if ["info", key] not in METAFILE_STD_KEYS:
                if logger:
                    logger("Removing key %r..." % ("info." + key,))
                del meta["info"][key]
                modified.add("info." + key)

        for idx, entry in enumerate(meta["info"].get("files", [])):
            for key in entry.keys():
                if ["info", "files", key] not in METAFILE_STD_KEYS:
                    if logger:
                        logger("Removing key %r from file #%d..." % (key, idx + 1))
                    del entry[key]
                    modified.add("info.files." + key)

            # Remove crap that certain PHP software puts in paths
            entry["path"] = [i for i in entry["path"] if i]

    return modified
161591
def emit_metadata_py(self, datadir, module_prefix):
        """"""

        if not os.path.isdir(datadir):
            os.mkdir(datadir)
        modulefilename = os.path.join(datadir, '__init__.py')

        # First, generate all of the individual per-region files in that directory
        for country_id in sorted(self.territory.keys()):
            filename = os.path.join(datadir, "region_%s.py" % country_id)
            self.emit_metadata_for_region_py(country_id, filename, module_prefix)

        # Same for any per-country-code alternate format files
        if self.alt_territory is not None:
            for country_code in sorted(self.alt_territory.keys()):
                filename = os.path.join(datadir, "alt_format_%s.py" % country_code)
                self.emit_alt_formats_for_cc_py(country_code, filename, module_prefix)

        # Now build a module file that includes them all
        with open(modulefilename, "w") as outfile:
            prnt(METADATA_FILE_PROLOG, file=outfile)
            prnt(COPYRIGHT_NOTICE, file=outfile)
            prnt(METADATA_FILE_IMPORT % {'module': module_prefix}, file=outfile)
            nongeo_codes = []
            country_codes = []
            for country_id in sorted(self.territory.keys()):
                terrobj = self.territory[country_id]
                if terrobj.o.id == REGION_CODE_FOR_NON_GEO_ENTITY:
                    nongeo_codes.append(country_id)  # int
                else:
                    country_codes.append("'%s'" % country_id)  # quoted string
            prnt("_AVAILABLE_REGION_CODES = [%s]" % ",".join(country_codes), file=outfile)
            if len(nongeo_codes) > 0:
                prnt("_AVAILABLE_NONGEO_COUNTRY_CODES = [%s]" % ", ".join(nongeo_codes), file=outfile)
            register_prefix = "short_" if self.short_data else ""
            prnt(METADATA_FILE_LOOP % {'prefix': register_prefix}, file=outfile)
            if len(nongeo_codes) > 0:
                prnt(METADATA_NONGEO_FILE_LOOP, file=outfile)

            if self.alt_territory is not None:
                for country_code in sorted(self.alt_territory.keys()):
                    prnt("from .alt_format_%s import PHONE_ALT_FORMAT_%s" % (country_code, country_code), file=outfile)
                prnt("_ALT_NUMBER_FORMATS = {%s}" %
                     ", ".join(["%s: PHONE_ALT_FORMAT_%s" % (cc, cc) for cc in sorted(self.alt_territory.keys())]),
                     file=outfile)

            # Build up a map from country code (int) to list of region codes (ISO 3166-1 alpha 2)
            country_code_to_region_code = {}
            for country_id in sorted(self.territory.keys()):
                terrobj = self.territory[country_id]
                if terrobj.o.country_code is not None:
                    country_code = int(terrobj.o.country_code)
                    if country_code not in country_code_to_region_code:
                        country_code_to_region_code[country_code] = []
                    if terrobj.o.main_country_for_code:
                        country_code_to_region_code[country_code].insert(0, terrobj.o.id)
                    else:
                        country_code_to_region_code[country_code].append(terrobj.o.id)

            # Emit the mapping from country code to region code if nonempty.
            if len(country_code_to_region_code.keys()) > 0:
                prnt(_COUNTRY_CODE_TO_REGION_CODE_PROLOG, file=outfile)
                prnt("_COUNTRY_CODE_TO_REGION_CODE = {", file=outfile)
                for country_code in sorted(country_code_to_region_code.keys()):
                    country_ids = country_code_to_region_code[country_code]
                    prnt('    %d: ("%s",),' % (country_code, '", "'.join(country_ids)), file=outfile)
                prnt("}", file=outfile)
972097
def validate_pages_json_data(d, preferred_lang):
    
    from .models import Page
    errors = []

    seen_complete_slugs = dict(
        (lang[0], set()) for lang in settings.PAGE_LANGUAGES)

    valid_templates = set(t[0] for t in settings.get_page_templates())
    valid_templates.add(settings.PAGE_DEFAULT_TEMPLATE)

    if d[JSON_PAGE_EXPORT_NAME] != JSON_PAGE_EXPORT_VERSION:
        return [_('Unsupported file version: %s') % repr(
            d[JSON_PAGE_EXPORT_NAME])], []
    pages = d['pages']
    for p in pages:
        # use the complete slug as a way to identify pages in errors
        slug = p['complete_slug'].get(preferred_lang, None)
        seen_parent = False
        for lang, s in p['complete_slug'].items():
            if lang not in seen_complete_slugs:
                continue
            seen_complete_slugs[lang].add(s)

            if '/' not in s: # root level, no parent req'd
                seen_parent = True
            if not seen_parent:
                parent_slug, ignore = s.rsplit('/', 1)
                if parent_slug in seen_complete_slugs[lang]:
                    seen_parent = True
                else:
                    parent = Page.objects.from_path(parent_slug, lang,
                        exclude_drafts=False)
                    if parent and parent.get_complete_slug(lang) == parent_slug:
                        # parent not included, but exists on site
                        seen_parent = True
            if not slug:
                slug = s

        if not slug:
            errors.append(_("%s has no common language with this site")
                % (p['complete_slug'].values()[0],))
            continue

        if not seen_parent:
            errors.append(_("%s did not include its parent page and a matching"
                " one was not found on this site") % (slug,))

        if p['template'] not in valid_templates:
            errors.append(_("%s uses a template not found on this site: %s")
                % (slug, p['template']))
            continue

        import_fields = set(p['content'].keys())
        import_fields |= set(('meta_title', 'meta_description', 'meta_keywords', 'meta_author', 'fb_page_type', 'fb_image'))
        template_fields = set(p.name for p in get_placeholders(p['template']) if
                p.name not in ('title', 'slug'))
        template_fields |= set(('meta_title', 'meta_description', 'meta_keywords', 'meta_author', 'fb_page_type', 'fb_image'))
        if template_fields != import_fields:
            errors.append(_("%s template contents are different than our "
                "template: %s") % (slug, p['template']))
            continue

    return errors
368549
public static void putMetaDatas(Map<String, Object> metadatas, String contentType, String fileName){
        metadatas.put(MULTIPART_FLAG, true);
        if(isNotBlank(contentType)) {
            metadatas.put(CONTENT_TYPE, contentType);
        }
        if(isNotBlank(fileName)) {
            metadatas.put(FILENAME, fileName);
        }
    }
1045098
def _get_metadata_as_string(self):
        """"""
        metalist = []
        for metaname, meta in iteritems(self.metadata):
            message = "Single value in metadata dictionary should be a list!"
            assert isinstance(meta, list), message
            for data in meta:
                if data:
                    metalist.append("!%s_%s = %s" % (self.geotype.capitalize(),
                                                     metaname, data))
        return "\n".join(metalist)