Compares two basis set dictionaries and prints a report about their differences
5182
def basis_comparison_report(bs1, bs2, uncontract_general=False):
    '''
    
    '''

    all_bs1 = list(bs1['elements'].keys())

    if uncontract_general:
        bs1 = manip.uncontract_general(bs1)
        bs2 = manip.uncontract_general(bs2)

    not_in_bs1 = []  # Found in bs2, not in bs1
    not_in_bs2 = all_bs1.copy()  # Found in bs1, not in bs2
    no_diff = []  # Elements for which there is no difference
    some_diff = []  # Elements that are different
    big_diff = []  # Elements that are substantially different

    for k, v in bs2['elements'].items():
        if k not in all_bs1:
            not_in_bs1.append(k)
            continue

        print()
        print("-------------------------------------")
        print(" Element ", k)
        bs1_el = bs1['elements'][k]

        max_rdiff_el = 0.0
        max_rdiff_ecp = 0.0

        # Check to make sure that neither or both have ecp/electron shells
        if 'electron_shells' in v and 'electron_shells' not in bs1_el:
            print("bs2 has electron_shells, but bs1 does not")
            max_rdiff_el = float('inf')
        if 'electron_shells' in bs1_el and 'electron_shells' not in v:
            print("bs1 has electron_shells, but bs2 does not")
            max_rdiff_el = float('inf')
        if 'ecp_potentials' in v and 'ecp_potentials' not in bs1_el:
            print("bs2 has ecp_potentials, but bs1 does not")
            max_rdiff_ecp = float('inf')
        if 'ecp_potentials' in bs1_el and 'ecp_potentials' not in v:
            print("bs1 has ecp_potentials, but bs2 does not")
            max_rdiff_ecp = float('inf')

        if 'electron_shells' in v and 'electron_shells' in bs1_el:
            max_rdiff_el = max(max_rdiff_el, shells_difference(v['electron_shells'], bs1_el['electron_shells']))
        if 'ecp_potentials' in v and 'ecp_potentials' in bs1_el:
            nel1 = v['ecp_electrons']
            nel2 = bs1_el['ecp_electrons']
            if int(nel1) != int(nel2):
                print('Different number of electrons replaced by ECP ({} vs {})'.format(nel1, nel2))
                max_rdiff_ecp = float('inf')
            else:
                max_rdiff_ecp = max(max_rdiff_ecp, potentials_difference(v['ecp_potentials'],
                                                                         bs1_el['ecp_potentials']))

        max_rdiff = max(max_rdiff_el, max_rdiff_ecp)

        # Handle some differences
        if max_rdiff == float('inf'):
            big_diff.append(k)
        elif max_rdiff == 0.0:
            no_diff.append(k)
        else:
            some_diff.append(k)

        not_in_bs2.remove(k)

    print()
    print("     Not in bs1: ", _print_list(not_in_bs1))
    print("     Not in bs2: ", _print_list(not_in_bs2))
    print("  No difference: ", _print_list(no_diff))
    print("Some difference: ", _print_list(some_diff))
    print(" BIG difference: ", _print_list(big_diff))
    print()

    return (len(not_in_bs1) == 0 and len(not_in_bs2) == 0 and len(some_diff) == 0 and len(big_diff) == 0)
1370407
def _get_values(cls, diff_dict, type='new'):
        '''
        
        '''
        ret_dict = {}
        for p in diff_dict.keys():
            if type in diff_dict[p].keys():
                ret_dict.update({p: diff_dict[p][type]})
            else:
                ret_dict.update(
                    {p: cls._get_values(diff_dict[p], type=type)})
        return ret_dict
187369
def _get_diffs(cls, dict1, dict2, ignore_missing_keys):
        '''
        
        '''
        ret_dict = {}
        for p in dict1.keys():
            if p not in dict2:
                ret_dict.update({p: {'new': dict1[p], 'old': cls.NONE_VALUE}})
            elif dict1[p] != dict2[p]:
                if isinstance(dict1[p], dict) and isinstance(dict2[p], dict):
                    sub_diff_dict = cls._get_diffs(dict1[p], dict2[p],
                                                   ignore_missing_keys)
                    if sub_diff_dict:
                        ret_dict.update({p: sub_diff_dict})
                else:
                    ret_dict.update({p: {'new': dict1[p], 'old': dict2[p]}})
        if not ignore_missing_keys:
            for p in dict2.keys():
                if p not in dict1.keys():
                    ret_dict.update({p: {'new': cls.NONE_VALUE,
                                         'old': dict2[p]}})
        return ret_dict
60997
def diffFromDefaults(theTask, report=False):
    
    # get the 2 dicts (trees: dicts of dicts)
    defaultTree = load(theTask, canExecute=False, strict=True, defaults=True)
    thisTree    = load(theTask, canExecute=False, strict=True, defaults=False)
    # they must be flattenable
    defaultFlat = cfgpars.flattenDictTree(defaultTree)
    thisFlat    = cfgpars.flattenDictTree(thisTree)
    # use the "set" operations till there is a dict.diff()
    # thanks to:  http://stackoverflow.com/questions/715234
    diffFlat = dict( set(thisFlat.items()) - \
                     set(defaultFlat.items()) )
    if report:
        defaults_of_diffs_only = {}
#       { k:defaultFlat[k] for k in diffFlat.keys() }
        for k in diffFlat:
            defaults_of_diffs_only[k] = defaultFlat[k]
        msg = 'Non-default values of "'+str(theTask)+'":\n'+ \
              _flat2str(diffFlat)+ \
              '\n\nDefault values:\n'+ \
              _flat2str(defaults_of_diffs_only)
        print(msg)
    return diffFlat
1325701
def __diff_dict(self,
                    level,
                    parents_ids=frozenset({}),
                    print_as_attribute=False,
                    override=False,
                    override_t1=None,
                    override_t2=None):
        """"""
        if override:
            # for special stuff like custom objects and named tuples we receive preprocessed t1 and t2
            # but must not spoil the chain (=level) with it
            t1 = override_t1
            t2 = override_t2
        else:
            t1 = level.t1
            t2 = level.t2

        if print_as_attribute:
            item_added_key = "attribute_added"
            item_removed_key = "attribute_removed"
            rel_class = AttributeRelationship
        else:
            item_added_key = "dictionary_item_added"
            item_removed_key = "dictionary_item_removed"
            rel_class = DictRelationship

        t1_keys = set(t1.keys())
        t2_keys = set(t2.keys())
        if self.ignore_string_type_changes or self.ignore_numeric_type_changes:
            t1_clean_to_keys = self.__get_clean_to_keys_mapping(keys=t1_keys, level=level)
            t2_clean_to_keys = self.__get_clean_to_keys_mapping(keys=t2_keys, level=level)
            t1_keys = set(t1_clean_to_keys.keys())
            t2_keys = set(t2_clean_to_keys.keys())
        else:
            t1_clean_to_keys = t2_clean_to_keys = None

        t_keys_intersect = t2_keys.intersection(t1_keys)

        t_keys_added = t2_keys - t_keys_intersect
        t_keys_removed = t1_keys - t_keys_intersect

        for key in t_keys_added:
            key = t2_clean_to_keys[key] if t2_clean_to_keys else key
            change_level = level.branch_deeper(
                notpresent,
                t2[key],
                child_relationship_class=rel_class,
                child_relationship_param=key)
            self.__report_result(item_added_key, change_level)

        for key in t_keys_removed:
            key = t1_clean_to_keys[key] if t1_clean_to_keys else key
            change_level = level.branch_deeper(
                t1[key],
                notpresent,
                child_relationship_class=rel_class,
                child_relationship_param=key)
            self.__report_result(item_removed_key, change_level)

        for key in t_keys_intersect:  # key present in both dicts - need to compare values
            key1 = t1_clean_to_keys[key] if t1_clean_to_keys else key
            key2 = t2_clean_to_keys[key] if t2_clean_to_keys else key
            item_id = id(t1[key1])
            if parents_ids and item_id in parents_ids:
                continue
            parents_ids_added = add_to_frozen_set(parents_ids, item_id)

            # Go one level deeper
            next_level = level.branch_deeper(
                t1[key1],
                t2[key2],
                child_relationship_class=rel_class,
                child_relationship_param=key)
            self.__diff(next_level, parents_ids_added)
332029
def block_similarity(self, block_a, block_b):
        

        # handle sim procedure blocks
        if self._project_a.is_hooked(block_a) and self._project_b.is_hooked(block_b):
            if self._project_a._sim_procedures[block_a] == self._project_b._sim_procedures[block_b]:
                return 1.0
            else:
                return 0.0

        try:
            block_a = NormalizedBlock(block_a, self._function_a)
        except (SimMemoryError, SimEngineError):
            block_a = None

        try:
            block_b = NormalizedBlock(block_b, self._function_b)
        except (SimMemoryError, SimEngineError):
            block_b = None

        # if both were None then they are assumed to be the same, if only one was the same they are assumed to differ
        if block_a is None and block_b is None:
            return 1.0
        elif block_a is None or block_b is None:
            return 0.0

        # get all elements for computing similarity
        tags_a = [s.tag for s in block_a.statements]
        tags_b = [s.tag for s in block_b.statements]
        consts_a = [c.value for c in block_a.all_constants]
        consts_b = [c.value for c in block_b.all_constants]
        all_registers_a = [s.offset for s in block_a.statements if hasattr(s, "offset")]
        all_registers_b = [s.offset for s in block_b.statements if hasattr(s, "offset")]
        jumpkind_a = block_a.jumpkind
        jumpkind_b = block_b.jumpkind

        # compute total distance
        total_dist = 0
        total_dist += _levenshtein_distance(tags_a, tags_b)
        total_dist += _levenshtein_distance(block_a.operations, block_b.operations)
        total_dist += _levenshtein_distance(all_registers_a, all_registers_b)
        acceptable_differences = self._get_acceptable_constant_differences(block_a, block_b)
        total_dist += _normalized_levenshtein_distance(consts_a, consts_b, acceptable_differences)
        total_dist += 0 if jumpkind_a == jumpkind_b else 1

        # compute similarity
        num_values = max(len(tags_a), len(tags_b))
        num_values += max(len(consts_a), len(consts_b))
        num_values += max(len(block_a.operations), len(block_b.operations))
        num_values += 1  # jumpkind
        similarity = 1 - (float(total_dist) / num_values)

        return similarity
793459
def isetdiff_flags(list1, list2):
    
    set2 = set(list2)
    return (item not in set2 for item in list1)
307620
def diff(self, diff):
        
        if diff is None:
            return None
        return dict(
            toVol=diff.toUUID,
            fromVol=diff.fromUUID,
            size=diff.size,
            sizeIsEstimated=diff.sizeIsEstimated,
        )
55619
def __diff(self, level, parents_ids=frozenset({})):
        """"""
        if level.t1 is level.t2:
            return

        if self.__skip_this(level):
            return

        if get_type(level.t1) != get_type(level.t2):
            report_type_change = True
            for type_group in self.ignore_type_in_groups:
                if self.type_check_func(level.t1, type_group) and self.type_check_func(level.t2, type_group):
                    report_type_change = False
                    break
            if report_type_change:
                self.__diff_types(level)
                return

        if isinstance(level.t1, strings):
            self.__diff_str(level)

        elif isinstance(level.t1, numbers):
            self.__diff_numbers(level)

        elif isinstance(level.t1, Mapping):
            self.__diff_dict(level, parents_ids)

        elif isinstance(level.t1, tuple):
            self.__diff_tuple(level, parents_ids)

        elif isinstance(level.t1, (set, frozenset, OrderedSet)):
            self.__diff_set(level)

        elif isinstance(level.t1, Iterable):
            if self.ignore_order:
                self.__diff_iterable_with_deephash(level)
            else:
                self.__diff_iterable(level, parents_ids)

        else:
            self.__diff_obj(level, parents_ids)
1366225
def compare_dicts(old=None, new=None):
    '''
    
    '''
    ret = {}
    for key in set((new or {})).union((old or {})):
        if key not in old:
            # New key
            ret[key] = {'old': '',
                        'new': new[key]}
        elif key not in new:
            # Key removed
            ret[key] = {'new': '',
                        'old': old[key]}
        elif new[key] != old[key]:
            # Key modified
            ret[key] = {'old': old[key],
                        'new': new[key]}
    return ret
381870
def blocks_with_differing_constants(self):
        
        differing_blocks = []
        diffs = dict()
        for (block_a, block_b) in self._block_matches:
            if self.blocks_probably_identical(block_a, block_b) and \
                    not self.blocks_probably_identical(block_a, block_b, check_constants=True):
                differing_blocks.append((block_a, block_b))
        for block_a, block_b in differing_blocks:
            ba = NormalizedBlock(block_a, self._function_a)
            bb = NormalizedBlock(block_b, self._function_b)
            diffs[(block_a, block_b)] = FunctionDiff._block_diff_constants(ba, bb)
        return diffs
896672
def compare(self, statement_a, statement_b):
        
        document_a = self.nlp(statement_a.text)
        document_b = self.nlp(statement_b.text)

        return document_a.similarity(document_b)
803908
def compare_dicts(dict1, dict2):
    
    for k, v in dict2.items():
        if v != dict1[k]:
            return False
    return True
153216
def list_to_compare_dict(self, list_form):
        """"""
        compare_dict = {}
        for field in list_form:
            if field['name'] in compare_dict:
                self.pr_dbg("List has duplicate field %s:\n%s" %
                            (field['name'], compare_dict[field['name']]))
                if compare_dict[field['name']] != field:
                    self.pr_dbg("And values are different:\n%s" % field)
                return None
            compare_dict[field['name']] = field
            for ign_f in self.mappings_ignore:
                compare_dict[field['name']][ign_f] = 0
        return compare_dict
561221
def _bsecurate_cli_compare_basis_files(args):
    ''''''
    ret = curate.compare_basis_files(args.file1, args.file2, args.readfmt1, args.readfmt2, args.uncontract_general)

    if ret:
        return "No difference found"
    else:
        return "DIFFERENCES FOUND. SEE ABOVE"
1357960
def difference(a: Iterable[Any], b: Container[Any]) -> List[Any]:
    
    d = []
    for item in a:
        if item not in b:
            d.append(item)
    return d
1268703
def compare_dict_iter(propval_a, propval_b, fs_a=None, fs_b=None,
                      options=None):
    
    if fs_a is None:
        fs_a = FieldSelector(tuple())
        fs_b = FieldSelector(tuple())
    if not options:
        options = DiffOptions()
    propvals = dict(a=propval_a, b=propval_b)
    values = dict()
    rev_keys = dict()
    for x in "a", "b":
        propval_x = propvals[x]
        vals = values[x] = set()
        rev_key = rev_keys[x] = dict()
        seen = collections.Counter()
        for k, v in collection_generator(propval_x):
            v = options.normalize_item(
                v, propval_a if options.duck_type else propval_x
            )
            if not v.__hash__:
                v = repr(v)
            if v is not _nothing or not options.ignore_empty_slots:
                vals.add((v, seen[v]))
                rev_key[(v, seen[v])] = k
                seen[v] += 1

    removed = values['a'] - values['b']
    added = values['b'] - values['a']

    if options.moved or options.unchanged:
        unchanged = values['a'] & values['b']
        for v, seq in unchanged:
            a_key = rev_keys['a'][v, seq]
            b_key = rev_keys['b'][v, seq]
            if options.moved and a_key != b_key:
                yield DiffInfo(
                    diff_type=DiffTypes.MOVED,
                    base=fs_a + [a_key],
                    other=fs_b + [b_key],
                )
            elif options.unchanged:
                yield DiffInfo(
                    diff_type=DiffTypes.NO_CHANGE,
                    base=fs_a + [a_key],
                    other=fs_b + [b_key],
                )

    removed_keys = set(rev_keys['a'][v, seq] for v, seq in removed)
    added_keys = set(rev_keys['b'][v, seq] for v, seq in added)
    modified_keys = set(removed_keys.intersection(added_keys))

    for v, seq in removed:
        a_key = rev_keys['a'][v, seq]
        if a_key in modified_keys:
            continue
        selector = fs_a + [a_key]
        yield DiffInfo(
            diff_type=DiffTypes.REMOVED,
            base=selector,
            other=fs_b,
        )

    for v, seq in added:
        b_key = rev_keys['b'][v, seq]
        if b_key in modified_keys:
            continue
        selector = fs_b + [b_key]
        yield DiffInfo(
            diff_type=DiffTypes.ADDED,
            base=fs_a,
            other=selector,
        )

    for key in modified_keys:
        yield DiffInfo(
            diff_type=DiffTypes.MODIFIED,
            base=fs_a + [key],
            other=fs_b + [key],
        )