Extra metadata for all products in a specific date
762114
def daily_metadata(year, month, day, dst_folder, writers=[file_writer], geometry_check=None,
                   num_worker_threads=1):
    

    threaded = False

    counter = {
        'products': 0,
        'saved_tiles': 0,
        'skipped_tiles': 0,
        'skipped_tiles_paths': []
    }

    if num_worker_threads > 1:
        threaded = True
        queue = Queue()

    # create folders
    year_dir = os.path.join(dst_folder, str(year))
    month_dir = os.path.join(year_dir, str(month))
    day_dir = os.path.join(month_dir, str(day))

    product_list = get_products_metadata_path(year, month, day)

    logger.info('There are %s products in %s-%s-%s' % (len(list(iterkeys(product_list))),
                                                       year, month, day))

    for name, product in iteritems(product_list):
        product_dir = os.path.join(day_dir, name)

        if threaded:
            queue.put([product, product_dir, counter, writers, geometry_check])
        else:
            counter = product_metadata(product, product_dir, counter, writers, geometry_check)

    if threaded:
        def worker():
            while not queue.empty():
                args = queue.get()
                try:
                    product_metadata(*args)
                except Exception:
                    exc = sys.exc_info()
                    logger.error('%s tile skipped due to error: %s' % (threading.current_thread().name,
                                                                       exc[1].__str__()))
                    args[2]['skipped_tiles'] += 1
                queue.task_done()

        threads = []
        for i in range(num_worker_threads):
            t = threading.Thread(target=worker)
            t.start()
            threads.append(t)

        queue.join()

    return counter
422864
def dateDict(date, prefix = ""):
    """"""
    res = {}

    res[prefix+"a"] = date.strftime("%a")
    res[prefix+"A"] = date.strftime("%A")
    res[prefix+"b"] = date.strftime("%b")
    res[prefix+"B"] = date.strftime("%B")
    res[prefix+"c"] = date.strftime("%c")
    res[prefix+"d"] = date.strftime("%d")
    res[prefix+"H"] = date.strftime("%H")
    res[prefix+"I"] = date.strftime("%I")
    res[prefix+"j"] = date.strftime("%j")
    res[prefix+"m"] = date.strftime("%m")
    res[prefix+"M"] = date.strftime("%M")
    res[prefix+"p"] = date.strftime("%p")
    res[prefix+"S"] = date.strftime("%S")
    res[prefix+"U"] = date.strftime("%U")
    res[prefix+"w"] = date.strftime("%w")
    res[prefix+"W"] = date.strftime("%W")
    res[prefix+"x"] = date.strftime("%x")
    res[prefix+"X"] = date.strftime("%X")
    res[prefix+"y"] = date.strftime("%y")
    res[prefix+"Y"] = date.strftime("%Y")
    res[prefix+"Z"] = date.strftime("%Z")

    for i, value in res.items():
        res[i] = locale_to_utf8(value)

    return res
711053
function setModifiedDate (subj, kb, doc) {
      if (SET_MODIFIED_DATES) {
        if (!getOption(tracker, 'trackLastModified')) return
        var deletions = kb.statementsMatching(subject, DCT('modified'))
        deletions = deletions.concat(kb.statementsMatching(subject, WF('modifiedBy')))
        var insertions = [ $rdf.st(subject, DCT('modified'), new Date(), doc) ]
        if (me) insertions.push($rdf.st(subject, WF('modifiedBy'), me, doc))
        updater.update(deletions, insertions, function (uri, ok, body) {})
      }
    }
624997
function timestampsPlugin(schema, options) {

  // Add the fields to the schema
  schema.add({
    createdAt: {
      type: Date,
      'default': Date.now
    },
    updatedAt: {
      type: Date,
      'default': Date.now
    },
    deletedAt: {
      type: Date,
      sparse: true
    }
  });

  // Define the pre save hook
  schema.pre('save', function (next) {
    this.updatedAt = new Date();
    next();
  });

  // Create an index on all the paths
  schema.path('createdAt').index(true);
  schema.path('updatedAt').index(true);
  schema.path('deletedAt').index(true);
}
1360576
def date_added(self, date_added):
        """"""
        self._indicator_data['dateAdded'] = self._utils.format_datetime(
            date_added, date_format='%Y-%m-%dT%H:%M:%SZ'
        )
532634
def get_products_metadata_path(year, month, day):
    

    products = {}
    path = 'products/{0}/{1}/{2}/'.format(year, month, day)
    for key in bucket.objects.filter(Prefix=path):
        product_path = key.key.replace(path, '').split('/')
        name = product_path[0]
        if name not in products:
            products[name] = {}

        if product_path[1] == 'metadata.xml':
            products[name]['metadata'] = key.key

        if product_path[1] == 'productInfo.json':
            products[name]['tiles'] = get_tile_metadata_path(key.key)

    return products
1326750
def per_version_data(self):
        
        ret = {}
        for cache_date in self.cache_dates:
            data = self._cache_get(cache_date)
            if len(data['by_version']) == 0:
                data['by_version'] = {'other': 0}
            ret[cache_date] = data['by_version']
        return ret
102777
def get_products(date)
      pdate = parse_date(date)
      unless pdate.nil?
        year, month, day = pdate
        prefix = "products/#{year}/#{month}/#{day}/"
        folder = "#{year}_#{month}_#{day}"
        objects = []

        @s3_client.list_objects(bucket: S3_BUCKET, prefix: prefix).each do |response|
          response.contents.each do |object|
            product_str = object.key.split(prefix)
            product = product_str[1].split('/')
            if product[1] == 'productInfo.json'
              objects << object.key
            end
          end
        end

        return objects
      else
        return []
      end
    end
131102
def _get_library_metadata(self, date_range):
        
        if date_range is None:
            raise Exception("A date range must be provided")
        if not (date_range.start and date_range.end):
            raise Exception("The date range {0} must contain a start and end date".format(date_range))

        start = date_range.start if date_range.start.tzinfo is not None else date_range.start.replace(tzinfo=mktz())
        end = date_range.end if date_range.end.tzinfo is not None else date_range.end.replace(tzinfo=mktz())
        query = {'$or': [{'start': {'$lte': start}, 'end': {'$gte': start}},
                         {'start': {'$gte': start}, 'end': {'$lte': end}},
                         {'start': {'$lte': end}, 'end': {'$gte': end}}]}

        cursor = self._collection.find(query,
                                       projection={'library_name': 1, 'start': 1, 'end': 1},
                                       sort=[('start', pymongo.ASCENDING)])

        results = []
        for res in cursor:
            start = res['start']
            if date_range.start.tzinfo is not None and start.tzinfo is None:
                start = start.replace(tzinfo=mktz("UTC")).astimezone(tz=date_range.start.tzinfo)

            end = res['end']
            if date_range.end.tzinfo is not None and end.tzinfo is None:
                end = end.replace(tzinfo=mktz("UTC")).astimezone(tz=date_range.end.tzinfo)

            results.append(TickStoreLibrary(res['library_name'], DateRange(start, end, CLOSED_CLOSED)))
        return results
241518
def __meta_metadata(self, field, key):
        """"""
        mf = ''
        try:
            mf = str([f[key] for f in self.metadata
                     if f['field_name'] == field][0])
        except IndexError:
            print("%s not in metadata field:%s" % (key, field))
            return mf
        else:
            return mf
752583
def get_last_date(self, field, filters_=[]):
        '''
            
        '''

        last_date = self.get_last_item_field(field, filters_=filters_)

        return last_date
1041120
def update_meta_data(meta=None):
    
    if meta is None:
        meta = {}
    if 'DATE' not in meta:
        meta['DATE'] = strftime("%Y-%m-%d %H:%M:%S", gmtime())
    if 'PROGRAM' not in meta:
        meta['PROGRAM'] = "AegeanTools.catalogs"
        meta['PROGVER'] = "{0}-({1})".format(__version__, __date__)
    return meta
237832
function mapXSetItems (items) {
  let results = [];
  for (let i = 0; i < items.length; i += 1) {
    results.push({
      timestamp: items[i][0],
      field: items[i][1][0],
      value: items[i][1][1]
    });
  }
  return results;
}
277328
def metadata_filter_as_string(metadata_filter):
    """"""
    if not isinstance(metadata_filter, dict):
        return metadata_filter

    additional = metadata_filter.get('additional', [])
    if additional == 'all':
        entries = ['all']
    else:
        entries = [key for key in additional if key not in _JUPYTEXT_CELL_METADATA]

    excluded = metadata_filter.get('excluded', [])
    if excluded == 'all':
        entries.append('-all')
    else:
        entries.extend(['-' + e for e in excluded])

    return ','.join(entries)
472654
def date(attr, metadata={})
      add_attribute(attr, :date, metadata)
      define_method("#{attr}=") do |arg|
        v = case arg
        when Date then arg
        when Time, DateTime then arg.to_date
        when String then Date.parse(arg)
        when Hash
          args = Util.extract_values(arg, :year, :month, :day)
          args.present? ? Date.new(*args.map(&:to_i)) : nil
        when nil then nil
        else raise ArgumentError.new("can't convert #{arg.class} to Date")
        end
        instance_variable_set("@#{attr}", v)
      end
    end
623931
def metadata(self):
        """"""
        auto = {u'records': self.size}
        auto.update(self.meta)
        return auto
832930
def info(self):
        """"""
        if self._composeinfo is not None:
            return self._composeinfo

        paths = [
            "metadata/composeinfo.json",
        ]
        self._composeinfo = self._load_metadata(paths, productmd.composeinfo.ComposeInfo)
        return self._composeinfo