Run in load_data module (which runs when file is loaded or dataframe is loaded),
  check for duplicate row/col names, and add index to names if necesary
774034
def main(net, df=None):
  '''
  
  '''
  if df is None:
    df = net.export_df()

  # rows
  #############
  rows = df.index.tolist()
  if type(rows[0]) is str:

    if len(rows) != len(list(set(rows))):
      new_rows = add_index_list(rows)
      df.index = new_rows

  elif type(rows[0]) is tuple:

    row_names = []
    for inst_row in rows:
      row_names.append(inst_row[0])

    if len(row_names) != len(list(set(row_names))):
      row_names = add_index_list(row_names)

      # add back to tuple
      new_rows = []
      for inst_index in range(len(rows)):
        inst_row = rows[inst_index]
        new_row = list(inst_row)
        new_row[0] = row_names[inst_index]
        new_row = tuple(new_row)
        new_rows.append(new_row)

      df.index = new_rows

  # cols
  #############
  cols = df.columns.tolist()
  if type(cols[0]) is str:

    # list column names
    if len(cols) != len(list(set(cols))):
      new_cols = add_index_list(cols)
      df.columns = new_cols

  elif type(cols[0]) is tuple:

    col_names = []
    for inst_col in cols:
      col_names.append(inst_col[0])

    if len(col_names) != len(list(set(col_names))):
      col_names = add_index_list(col_names)

      # add back to tuple
      new_cols = []
      for inst_index in range(len(cols)):
        inst_col = cols[inst_index]
        new_col = list(inst_col)
        new_col[0] = col_names[inst_index]
        new_col = tuple(new_col)
        new_cols.append(new_col)

      df.columns = new_cols

  # return dataframe with unique names
  return df
394837
def _check_field_names(self, field_names):
        
        if field_names:
            for field_name in field_names:
                try:
                    self.column[field_name]
                except KeyError:
                    raise InvalidIndexError('Trying to use non indexed field "%s"' % field_name)
476684
def _add_row(self, index):
        
        self._index.append(index)
        for c, _ in enumerate(self._columns):
            self._data[c].append(None)
531278
def insert(self, insert_index, val):
        

        def prevent_duplicates(index, val):
            raise BaseLunrException("Duplicate index")

        self.upsert(insert_index, val, prevent_duplicates)
711705
def drop_duplicate_cols(df):
    
    names = set(df.columns)
    for n in names:
        if len(df[n].shape) > 1:
            # get subset of df w/ colname n
            sub = df[n]
            # make unique colnames
            sub.columns = ['%s-%s' % (n, x) for x in range(sub.shape[1])]
            # get colname w/ max # of data
            keep = sub.count().idxmax()
            # drop all columns of name n from original df
            del df[n]
            # update original df w/ longest col with name n
            df[n] = sub[keep]

    return df
596459
def load_adjusted_array(self, domain, columns, dates, sids, mask):
        
        if len(columns) != 1:
            raise ValueError(
                "Can't load multiple columns with DataFrameLoader"
            )

        column = columns[0]
        self._validate_input_column(column)

        date_indexer = self.dates.get_indexer(dates)
        assets_indexer = self.assets.get_indexer(sids)

        # Boolean arrays with True on matched entries
        good_dates = (date_indexer != -1)
        good_assets = (assets_indexer != -1)

        data = self.baseline[ix_(date_indexer, assets_indexer)]
        mask = (good_assets & as_column(good_dates)) & mask

        # Mask out requested columns/rows that didn't match.
        data[~mask] = column.missing_value

        return {
            column: AdjustedArray(
                # Pull out requested columns/rows from our baseline data.
                data=data,
                adjustments=self.format_adjustments(dates, sids),
                missing_value=column.missing_value,
            ),
        }
914796
def _prepend_row_index(rows, index):
    """"""
    if index is None or index is False:
        return rows
    if len(index) != len(rows):
        print('index=', index)
        print('rows=', rows)
        raise ValueError('index must be as long as the number of data rows')
    rows = [[v] + list(row) for v, row in zip(index, rows)]
    return rows
823337
def count_rows_duplicates(self, table, cols='*'):
        """"""
        return self.count_rows(table, '*') - self.count_rows_distinct(table, cols)
1122620
def set_row_names(self, row_names):
        
        if self.row_count() != len(row_names) or len(self._row_name_list) > 0 or len(self._row_name_idx) > 0:
            raise NotImplementedError("You can only manually set names once data has been added")

        for idx, k in enumerate(row_names):
            self._row_name_idx[k] = idx

        self._row_name_list = row_names
1342861
def __loadindcomps(self):
        '''  '''
        csv_path = os.path.join(os.path.dirname(__file__), self.stock_no_files)
        with open(csv_path) as csv_file:
            csv_data = csv.reader(csv_file)
            result = {}
            check_words = re.compile(r'^[\d]{2,}[\w]?')
            for i in csv_data:
                if check_words.match(i[2]):
                    try:
                        result[i[2]].append(i[0].decode('utf-8'))
                    except (ValueError, KeyError):
                        try:
                            result[i[2]] = [i[0].decode('utf-8')]
                        except KeyError:
                            pass
            return result
260475
def check_duplicate_filenames(files):
    """"""
    # Check whether we'll have duplicates
    used_names = set()
    dup_names = list()

    for this_file in files:
        this_fname = os.path.basename(this_file)
        if this_fname in used_names:
            dup_names.append(this_file)
        else:
            used_names.add(this_fname)

    if len(dup_names) > 0:
        logger.warning(
            'Duplicate file name(s) found. Having duplicate file names will '
            'break some links. List of files: {}'.format(sorted(dup_names),))
623466
def indexcol(self, col: str):
        
        try:
            self.df[col] = self.df.index.values
        except Exception as e:
            self.err(e)
            return
        self.ok("Column", col, "added from the index")
397131
def add_row(self, label, row_data, columns=""):
        
        # use provided column order, making sure you don't lose any values
        # from self.df.columns
        if len(columns):
            if sorted(self.df.columns) == sorted(columns):
                self.df.columns = columns
            else:
                new_columns = []
                new_columns.extend(columns)
                for col in self.df.columns:
                    if col not in new_columns:
                        new_columns.append(col)
        # makes sure all columns have data or None
        if sorted(row_data.keys()) != sorted(self.df.columns):
            # add any new column names
            for key in row_data:
                if key not in self.df.columns:
                    self.df[key] = None
            # add missing column names into row_data
            for col_label in self.df.columns:
                if col_label not in list(row_data.keys()):
                    row_data[col_label] = None

        # (make sure you are working with strings)
        self.df.index = self.df.index.astype(str)
        label = str(label)

        # create a new row with suffix "new"
        # (this ensures that you get a unique, new row,
        #  instead of adding on to an existing row with the same label)
        self.df.loc[label + "new"] = pd.Series(row_data)
        # rename it to be correct
        self.df.rename(index={label + "new": label}, inplace=True)
        # use next line to sort index inplace
        #self.df.sort_index(inplace=True)
        return self.df
981554
def reload(self):
        ''
        self.rows = []
        for r in self.iterload():
            self.addRow(r)
1020281
def load_data(self, df):
        
        stmt = ddl.LoadData(self._qualified_name, df)
        return self._execute(stmt)
596040
def load(self, filename=None):
        '''
        
        '''
        fields = []
        with open(filename, 'r') as f:
            format_data = f.read().strip()

        lines = format_data.split('\n')
        self._sql_version = lines.pop(0)
        self._num_fields = int(lines.pop(0))

        for line in lines:
            # Get rid of mulitple spaces
            line = re.sub(' +', ' ', line.strip())
            row_format = BCPFormatRow(line.split(' '))
            fields.append(row_format)

        self.fields = fields
        self.filename = filename
677805
def reload_sync(self):
        ''
        header_lines = options.get('header', self)
        delim = options.get('delimiter', self)

        with self.source.open_text() as fp:
            # get one line anyway to determine number of columns
            lines = list(getlines(fp, int(header_lines) or 1))
            headers = [L.split(delim) for L in lines]

            if header_lines <= 0:
                self.columns = [ColumnItem('', i) for i in range(len(headers[0]))]
            else:
                self.columns = [
                    ColumnItem('\\n'.join(x), i)
                        for i, x in enumerate(zip(*headers[:header_lines]))
                    ]

            lines = lines[header_lines:]  # in case of header_lines == 0
            self._rowtype = namedlist('tsvobj', [c.name for c in self.columns])

            self.recalc()
            self.rows = []

            with Progress(total=self.source.filesize) as prog:
                for L in itertools.chain(lines, getlines(fp)):
                    row = L.split(delim)
                    ncols = self._rowtype.length()  # current number of cols
                    if len(row) > ncols:
                        # add unnamed columns to the type not found in the header
                        newcols = [ColumnItem('', len(row)+i, width=8) for i in range(len(row)-ncols)]
                        self._rowtype = namedlist(self._rowtype.__name__, list(self._rowtype._fields) + ['_' for c in newcols])
                        for c in newcols:
                            self.addColumn(c)
                    elif len(row) < ncols:
                        # extend rows that are missing entries
                        row.extend([None]*(ncols-len(row)))

                    self.addRow(self._rowtype(row))
                    prog.addProgress(len(L))