Converts nested dictionary filters into django-style key value pairs

    Map filter operators and aliases to operator-land
    Additionally, perform replacements according to attribute map
    Automatically assumes __eq if not explicitly defined
518348
def _normalise_key_values(filter_obj, attr_map=None):
    
    new_filter = {}
    for key, constraints in filter_obj.items():
        aliased_key = key
        if attr_map is not None:
            aliased_key = attr_map.get(key)
            if aliased_key is None:
                raise CloudValueError(
                    'Invalid key %r for filter attribute; must be one of:\n%s' % (
                        key,
                        attr_map.keys()
                    )
                )
        if not isinstance(constraints, dict):
            constraints = {'eq': constraints}
        for operator, value in constraints.items():
            # FIXME: deprecate this $ nonsense
            canonical_operator = FILTER_OPERATOR_ALIASES.get(operator.lstrip('$'))
            if canonical_operator is None:
                raise CloudValueError(
                    'Invalid operator %r for filter key %s; must be one of:\n%s' % (
                        operator,
                        key,
                        FILTER_OPERATOR_ALIASES.keys()
                    )
                )
            canonical_key = str('%s__%s' % (aliased_key, canonical_operator))
            new_filter[canonical_key] = _normalise_value(value)
    return new_filter
1332798
def clean_map(obj: Mapping[Any, Any]) -> Mapping[Any, Any]:
    
    return {k: v for k, v in obj.items() if v is not None}
781016
def get_filter_qobj(self, keys=None):
    
    # only care about columns in aggregates right?
    cols = set()
    for agg in self.select.aggregates:
        cols.update(agg.cols)
    sels = [SelectExpr(col, [col], col, None) for col in cols]
    select = Select(sels)
    where = list(self.where)
    if keys:
      keys = list(keys)
      keys = map(sqlize, list(keys))
      expr = self.select.nonaggs[0].expr

      clause = []
      if None in keys:
        clause.append("%s is null" % expr)
      if len([k for k in keys if k is not None]) > 0:
        clause.append("%s in %%s" % expr)
      clause = " or ".join(clause)
      where.append(clause)
    else:
      where.append( '%s = %%s' % (self.select.nonaggs[0].expr ) )
    q = Query(self.db, select, self.fr, where)
    return q
336163
def Compile(self, filter_implementation):
    """"""
    arguments = [self.attribute]
    for argument in self.args:
      arguments.append(argument.Compile(filter_implementation))
    expander = filter_implementation.FILTERS['ValueExpander']
    context_cls = filter_implementation.FILTERS['Context']
    return context_cls(arguments=arguments,
                       value_expander=expander)
1072745
def _init_filters(self):
        """"""
        self.filters = {}

        @self.template_filter()
        def format_ts(value, format_='%a, %b %d %Y %H:%M:%S'):
            """Formats the supplied timestamp using format_

            :param str value: The timestamp to be formatted
            :param str format_:  The format string
            :return: The correctly formatted timestamp as determined by format_
            :rtype: str
            """
            if format_ == '%s':
                return timestamp_to_sec(value)
            else:
                value = timestamp_to_datetime(value)
                return value.strftime(format_)

        @self.template_filter('urlsplit')
        def get_urlsplit(url):
            """Splits the supplied URL

            :param str url: The url to be split
            :return: The split url
            :rtype: urllib.parse.SplitResult
            """
            split = urlsplit(url)
            return split

        @self.template_filter()
        def tojson(obj):
            """Converts the supplied object/array/any to a JSON string if it can be JSONified

            :param any obj: The value to be converted to a JSON string
            :return: The JSON string representation of the supplied value
            :rtype: str
            """
            return json.dumps(obj)

        @self.template_filter()
        def tobool(bool_val):
            """Converts a python boolean to a JS "true" or "false" string
            :param any obj: A value to be evaluated as a boolean
            :return: The string "true" or "false" to be inserted into JS
            """

            return 'true' if bool_val else 'false'
1155925
def _from_dict(cls, mapping):
        
        if mapping.get("all"):
            return cls(all_=True)

        r_mappings = mapping.get("ranges", ())
        ranges = [KeyRange(**r_mapping) for r_mapping in r_mappings]

        return cls(keys=mapping.get("keys", ()), ranges=ranges)
1090394
def clean_query_Dict(cls, query_Dict):
    
    return {k: v for k, v in query_Dict.items() if v}
422115
function normFilter(model, filter, fieldName) {
  if (!filter) return null;

  let result = {},
    fields = model.def.fields;
  for (const key in filter) {
    if (key === '$and' || key === '$or') { // {$and:[]}, {$or: []}
      const subfilters = [];
      for (const j in filter[key]) {
        const sub = normFilter(model, filter[key][j]);
        if (!sub) continue;
        subfilters.push(sub);
      }
      if (subfilters.length > 0) result[key] = subfilters;
      continue;
    }
    if (key.charAt(0) === '$') { // key is an operator, {op: value}
      if (!fieldName) {
        throw new QueryError(model.name, 'no field for operator %s in filter', key);
      }
      if (!fields[fieldName]) {
        throw new QueryError(model.name, 'nonexistent field %s in filter', fieldName);
      }
      result[key] = fields[fieldName].normalize(filter[key], { query: true });
      continue;
    }

    // key is a field, {field: value}, {field: {op: value, ...}}
    fieldName = key;
    const field = fields[key];
    if (!field) throw new QueryError(model.name, 'nonexistent field %s in filter', key);
    if (field.props.filterRedirect) {
      const proxyFields = field.props.filterRedirect.replace(/\s+/g, '').split(',');
      const proxyFilter = { $or: [] };
      for (const pi in proxyFields) {
        const proxyFieldName = proxyFields[pi];
        const fieldProxyFilter = {};
        fieldProxyFilter[proxyFieldName] = filter[key];
        proxyFilter.$or.push(fieldProxyFilter);
      }
      Object.assign(result, normFilter(model, proxyFilter));
      continue;
    }

    // {field: {op: value, ...}}
    if (util.isPlainObject(filter[key])) {
      result[fieldName] = normFilter(model, filter[key], fieldName);
      continue;
    }

    // {field: value}, value is not string
    let value = filter[key];
    if (typeof value !== 'string') { // array, date, int, float, bool, etc
      result[fieldName] = field.normalize(value, { query: true });
      continue;
    }

    // {field: value}, value is a plain string
    value = value.trim();
    if (!(value.substring(0, 1) === '{' && value.substr(-1) === '}')) {
      result[fieldName] = field.normalize(value, { query: true });
      continue;
    }

    // {field: value}, value is string and may contain a json object which is a query filter too
    const jsonExpr = null;
    try {
      eval('jsonExpr = '+value);//eslint-disable-line
    } catch (e) {}// eslint-disable-line no-empty
    if (!jsonExpr) { // parse fails, treat as plain string
      result[fieldName] = field.normalize(value, { query: true });
    } else {
      const jsonFilter = {};
      for (const jk in jsonExpr) {
        if (jk.charAt(0) === '$') { // expand {op:val} to {field:{op: val}}
          jsonFilter[fieldName] = {};
          jsonFilter[fieldName][jk] = jsonExpr[jk];
        } else { // {field: value}
          jsonFilter[jk] = jsonExpr[jk];
        }
      }
      result = Object.assign(result, normFilter(model, jsonFilter, fieldName));
    }
  }

  if (util.isEmptyObject(result)) return null;
  return result;
}
545668
def _Q_to_filter(self, q):
        
        default_filter = ANDFilter
        if q.connector == "OR":
            default_filter = ORFilter
        filters = []
        for child in q.children:
            if isinstance(child, Q):
                if child.children:
                    filters.append(self._Q_to_filter(child))
            elif isinstance(child, tuple):
                field, value = child
                filters.append(self._build_inner_filter(field, value))
        if len(filters) == 1:
            filter = filters[0]
            if q.negated:
                return NotFilter(filter)
            return filter
        if q.negated:
            return NotFilter(default_filter(filters))
        return default_filter(filters)
115162
def keyfilter(predicate, d, factory=dict):
    
    rv = factory()
    for k, v in iteritems(d):
        if predicate(k):
            rv[k] = v
    return rv
397329
def to_flat_dict(self, **kwargs):
        
        if kwargs:
            return self.filter(**kwargs).to_flat_dict()
        return {param.uniquetwig: param for param in self._params}
1370111
def get_attr_filters(self):
        
        for f in self.data.keys():
            if f not in self.multi_attrs:
                continue
            fv = self.data[f]
            if isinstance(fv, dict):
                fv['key'] = f
            else:
                fv = {f: fv}
            vf = ValueFilter(fv)
            vf.annotate = False
            yield vf
1186711
def create_attr_filter(request, mapped_class):
    

    mapping = {
        'eq': '__eq__',
        'ne': '__ne__',
        'lt': '__lt__',
        'lte': '__le__',
        'gt': '__gt__',
        'gte': '__ge__',
        'like': 'like',
        'ilike': 'ilike'
    }
    filters = []
    if 'queryable' in request.params:
        queryable = request.params['queryable'].split(',')
        for k in request.params:
            if len(request.params[k]) <= 0 or '__' not in k:
                continue
            col, op = k.split("__")
            if col not in queryable or op not in mapping:
                continue
            column = getattr(mapped_class, col)
            f = getattr(column, mapping[op])(request.params[k])
            filters.append(f)
    return and_(*filters) if len(filters) > 0 else None
1296409
def mapped(cls, key='id', **kwargs):
    """"""
    kwargs.setdefault('metadata', {})
    kwargs['metadata']['jsonldPredicate'] = {'mapSubject': key}
    kwargs.setdefault('default', attr.Factory(list))

    def converter(value):
        """Convert mapping to a list of instances."""
        if isinstance(value, dict):
            result = []
            for k, v in iteritems(value):
                if not hasattr(cls, 'from_cwl'):
                    vv = dict(v)
                    vv[key] = k
                else:
                    vv = attr.evolve(cls.from_cwl(v), **{key: k})
                result.append(vv)
        else:
            result = value

        def fix_keys(data):
            """Fix names of keys."""
            for a in fields(cls):
                a_name = a.name.rstrip('_')
                if a_name in data:
                    yield a.name, data[a_name]

        return [
            cls(**{kk: vv
                   for kk, vv in fix_keys(v)}) if not isinstance(v, cls) else v
            for v in result
        ]

    kwargs['converter'] = converter
    return attr.ib(**kwargs)
590765
def to_dict(self):
        
        return {k: getattr(self, k) for k in filter(
            lambda k: not k.startswith('_') and k != 'to_dict', dir(self))}
1148302
def _build_query_dict(self, formdata=None):
        
        if self.is_valid() and formdata is None:
            formdata = self.cleaned_data
        key = "{field}__{operator}".format(**formdata)
        if formdata['operator'] == "isnull":
            return {key: None}
        elif formdata['operator'] == "istrue":
            return {formdata['field']: True}
        elif formdata['operator'] == "isfalse":
            return {formdata['field']: False}
        return {key: formdata['value']}
1112895
def filter_to_dmql(filter_dict):
        """"""

        if not isinstance(filter_dict, (dict, collections.OrderedDict)):
            raise TypeError('Expected a dictionary type buy got {} instead.'.format(type(filter_dict)))

        def is_date_time_type(val):
            """Returns True if the value is a datetime"""
            return isinstance(val, (datetime.datetime, datetime.date, datetime.time))

        def evaluate_datetime(val):
            """Converts the datetime object into the RETS expected format"""
            date_format = '%Y-%m-%d'
            time_format = '%H:%M:%S'
            datetime_format = '{}T{}'.format(date_format, time_format)

            if isinstance(val, datetime.datetime):
                evaluated = val.strftime(datetime_format)
            elif isinstance(val, datetime.date):
                evaluated = val.strftime(date_format)
            elif isinstance(val, datetime.time):
                evaluated = val.strftime(time_format)
            else:
                evaluated = val

            return evaluated

        def evaluate_operators(key_dict):
            """Turns the custom filter operators into the expected RETS query"""
            allowed_operators = ['$gte', '$lte', '$contains', '$begins', '$ends', '$in', '$nin', '$neq']

            # If key not in allowed_operators, assume it is a field name with the and operation.
            if not all(op in allowed_operators for op in key_dict.keys()):
                raise ValueError("You have supplied an invalid operator. "
                                 "Please provide one of the following {}".format(allowed_operators))

            # We can have a single operator key, or the combination of gte/lte
            keys = key_dict.keys()
            string = ''

            # Search between two numbers or two dates
            if len(keys) == 2 and all(k in ['$gte', '$lte'] for k in keys):
                if all(is_date_time_type(key_dict[v]) for v in keys):
                    # comparing dates
                    string = '{}-{}'.format(evaluate_datetime(key_dict['$gte']), evaluate_datetime(key_dict['$lte']))
                else:
                    # comparing numbers
                    try:
                        float(key_dict['$gte'])
                        float(key_dict['$lte'])
                    except ValueError:
                        raise ValueError("$gte and $lte expect numeric or datetime values")
                    string = '{:.2f}-{:.2f}'.format(key_dict['$gte'], key_dict['$lte'])

            # Using a single operator key
            elif len(keys) == 1:
                if '$gte' in key_dict:
                    if is_date_time_type(key_dict['$gte']):
                        string = '{}+'.format(evaluate_datetime(key_dict['$gte']))
                    else:
                        try:
                            float(key_dict['$gte'])
                        except ValueError:
                            raise ValueError("$gte expects a numeric value or a datetime object")
                        string = '{:.2f}+'.format(key_dict['$gte'])

                elif '$lte' in key_dict:
                    if is_date_time_type(key_dict['$lte']):
                        string = '{}-'.format(evaluate_datetime(key_dict['$lte']))
                    else:
                        try:
                            float(key_dict['$lte'])
                        except ValueError:
                            raise ValueError("$lte expects a numeric value or a datetime object")
                        string = '{:.2f}-'.format(key_dict['$lte'])

                elif '$in' in key_dict:
                    if not isinstance(key_dict['$in'], list):
                        raise ValueError("$in expects a list of strings")
                    key_dict['$in'] = [evaluate_datetime(v) for v in key_dict['$in']]
                    if not all(isinstance(v, string_types) for v in key_dict['$in']):
                        raise ValueError("$in expects a list of strings")
                    options = ','.join(key_dict['$in'])
                    string = '{}'.format(options)

                elif '$nin' in key_dict:
                    if not isinstance(key_dict['$nin'], list):
                        raise ValueError("$nin expects a list of strings")
                    key_dict['$nin'] = [evaluate_datetime(v) for v in key_dict['$nin']]
                    if not all(isinstance(v, string_types) for v in key_dict['$nin']):
                        raise ValueError("$nin expects a list of strings")
                    options = ','.join(key_dict['$nin'])
                    string = '~{}'.format(options)

                elif '$contains' in key_dict:
                    if not isinstance(key_dict['$contains'], string_types):
                        raise ValueError("$contains expects a string.")
                    string = '*{}*'.format(key_dict['$contains'])

                elif '$begins' in key_dict:
                    if not isinstance(key_dict['$begins'], string_types):
                        raise ValueError("$begins expects a string.")
                    string = '{}*'.format(key_dict['$begins'])

                elif '$ends' in key_dict:
                    if not isinstance(key_dict['$ends'], string_types):
                        raise ValueError("$ends expects a string.")
                    string = '*{}'.format(key_dict['$ends'])

                elif '$neq' in key_dict:
                    string = '~{}'.format(key_dict['$neq'])

            else:
                # Provided too many or too few operators
                raise ValueError("Please supply $gte and $lte for getting values between numbers or 1 of {}".format(
                    allowed_operators))

            return string

        dmql_search_filters = []

        for filt, value in filter_dict.items():
            dmql_string = '({}='.format(filt)
            if isinstance(value, dict):
                # Applying an operator. This will need to be recursive because of the or possibility
                dmql_string += evaluate_operators(key_dict=value)
            else:
                # Simle equals statement
                dmql_string += '{}'.format(evaluate_datetime(value))
            dmql_string += ')'
            dmql_search_filters.append(dmql_string)

        search_string = ','.join(dmql_search_filters)
        # Converts the filter dictionary to dmqp string
        logger.debug("Filter returned the following DMQL: {}".format(search_string))
        return search_string