Partitioned tables
2594
@Override
  public boolean shouldCreateWorkunit(Partition sourcePartition, LongWatermark lowWatermark) {
    // If a whitelist is provided only create workunits for those partitions
    if (!this.partitionsWhitelist.isEmpty()) {
      return this.partitionsWhitelist.contains(sourcePartition.getCompleteName());
    }
    // If no whitelist is set, all partitions of a dataset are backfilled
    return true;
  }
943646
@Override
	public void createPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogPartition partition, boolean ignoreIfExists)
			throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, PartitionAlreadyExistsException, CatalogException {
		throw new UnsupportedOperationException();
	}
1311228
void createHuffmanDecodingTables() {
        final int alphabetSize = this.alphabetSize;

        for (int table = 0; table < tableCodeLengths.length; table++) {
            final int[] tableBases = codeBases[table];
            final int[] tableLimits = codeLimits[table];
            final int[] tableSymbols = codeSymbols[table];
            final byte[] codeLengths = tableCodeLengths[table];

            int minimumLength = HUFFMAN_DECODE_MAX_CODE_LENGTH;
            int maximumLength = 0;

            // Find the minimum and maximum code length for the table
            for (int i = 0; i < alphabetSize; i++) {
                final byte currLength = codeLengths[i];
                maximumLength = Math.max(currLength, maximumLength);
                minimumLength = Math.min(currLength, minimumLength);
            }
            minimumLengths[table] = minimumLength;

            // Calculate the first output symbol for each code length
            for (int i = 0; i < alphabetSize; i++) {
                tableBases[codeLengths[i] + 1]++;
            }
            for (int i = 1, b = tableBases[0]; i < HUFFMAN_DECODE_MAX_CODE_LENGTH + 2; i++) {
                b += tableBases[i];
                tableBases[i] = b;
            }

            // Calculate the first and last Huffman code for each code length (codes at a given
            // length are sequential in value)
            for (int i = minimumLength, code = 0; i <= maximumLength; i++) {
                int base = code;
                code += tableBases[i + 1] - tableBases[i];
                tableBases[i] = base - tableBases[i];
                tableLimits[i] = code - 1;
                code <<= 1;
            }

            // Populate the mapping from canonical code index to output symbol
            for (int bitLength = minimumLength, codeIndex = 0; bitLength <= maximumLength; bitLength++) {
                for (int symbol = 0; symbol < alphabetSize; symbol++) {
                    if (codeLengths[symbol] == bitLength) {
                        tableSymbols[codeIndex++] = symbol;
                    }
                }
            }
        }

        currentTable = selectors[0];
    }
190450
def _partition_index_names(provisioned_index_names, index_names):
    ''''''
    existing_index_names = set()
    new_index_names = set()
    for name in index_names:
        if name in provisioned_index_names:
            existing_index_names.add(name)
        else:
            new_index_names.add(name)
    index_names_to_be_deleted = provisioned_index_names - existing_index_names
    return existing_index_names, new_index_names, index_names_to_be_deleted
872577
def tables_with_counts(self):
        """"""
        table_to_count = lambda t: self.count_rows(t)
        return zip(self.tables, map(table_to_count, self.tables))
1152273
def partition(partition):
    """"""
    nodes = set()
    for part in partition:
        for node in part:
            if node in nodes:
                raise ValueError(
                    'Micro-element {} may not be partitioned into multiple '
                    'macro-elements'.format(node))
            nodes.add(node)
387921
def partition_pair(bif_point):
    ''''''
    n = float(sum(1 for _ in bif_point.children[0].ipreorder()))
    m = float(sum(1 for _ in bif_point.children[1].ipreorder()))
    return (n, m)
347594
def _add_partition(self, connection, partition):
        
        logger.debug('Creating foreign table for partition.\n    partition: {}'.format(partition.name))
        with connection.cursor() as cursor:
            postgres_med.add_partition(cursor, partition.datafile, partition.vid)
911598
@Override
	public void open() {
		synchronized (stateLock) {
			if (!closed) {
				throw new IllegalStateException("currently not closed.");
			}
			closed = false;
		}
		
		// create the partitions
		final int partitionFanOut = getPartitioningFanOutNoEstimates(this.availableMemory.size()); 
		createPartitions(partitionFanOut);
		
		// set up the table structure. the write behind buffers are taken away, as are one buffer per partition
		final int numBuckets = getInitialTableSize(this.availableMemory.size(), this.segmentSize, 
			partitionFanOut, this.avgRecordLen);
		
		initTable(numBuckets, (byte) partitionFanOut);
	}
331857
private int partitionItems(int left, int right, int pivot, double[] origin) {
		double pivotDistance = distance(origin, items[pivot]);
		listSwap(items, pivot, right - 1);
		listSwap(indexes, pivot, right - 1);
		int storeIndex = left;
		for (int i = left; i < right - 1; i++) {
			if (distance(origin, items[i]) <= pivotDistance) {
				listSwap(items, i, storeIndex);
				listSwap(indexes, i, storeIndex);
				storeIndex++;
			}
		}
		listSwap(items, storeIndex, right - 1);
		listSwap(indexes, storeIndex, right - 1);
		return storeIndex;
	}
730649
def _add_partition(self, connection, partition):
        
        logger.debug('Creating virtual table for partition.\n    partition: {}'.format(partition.name))
        sqlite_med.add_partition(connection, partition.datafile, partition.vid+'_vt')
96807
private void mergeInsertedTables() {
        int i=0, j=0;
        float [] lNewWidths = null;
        int [] lDummyWidths = new int[columns];     // to keep track in how many new cols this one will be split
        float[][] lDummyColumnWidths = new float[columns][]; // bugfix Tony Copping
        int [] lDummyHeights = new int[rows.size()]; // to keep track in how many new rows this one will be split
        ArrayList<Row> newRows = null;
        boolean isTable=false;
        int lTotalRows  = 0, lTotalColumns      = 0;
        int lNewMaxRows = 0, lNewMaxColumns     = 0;
        
        Table lDummyTable = null;
        
        // first we'll add new columns when needed
        // check one column at a time, find maximum needed nr of cols
        // Search internal tables and find one with max columns
        for (j=0; j < columns; j++) {
            lNewMaxColumns = 1; // value to hold in how many columns the current one will be split
            float [] tmpWidths = null;
            for (i=0; i < rows.size(); i++) {
                if ( Table.class.isInstance(((Row) rows.get(i)).getCell(j)) ) {
                    isTable=true;
                    lDummyTable = ((Table) ((Row) rows.get(i)).getCell(j));
                    if( tmpWidths == null) {
                        tmpWidths = lDummyTable.widths;
                        lNewMaxColumns=tmpWidths.length;
                    }
                    else {
                        int cols = lDummyTable.getDimension().width;
                        float [] tmpWidthsN = new float[ cols * tmpWidths.length];
                        float tpW=0, btW=0, totW=0;
                        int tpI=0, btI=0, totI=0;
                        tpW+=tmpWidths[0];
                        btW+=lDummyTable.widths[0];
                        while( tpI<tmpWidths.length && btI<cols) {
                            if( btW>tpW) {
                                tmpWidthsN[totI] = tpW-totW;
                                tpI++;
                                if(tpI<tmpWidths.length) {
                                    tpW+=tmpWidths[tpI];
                                }
                            }
                            else {
                                tmpWidthsN[totI] = btW-totW;
                                btI++;
                                if(Math.abs(btW - tpW) < 0.0001) {
                                    tpI++;
                                    if(tpI<tmpWidths.length) {
                                        tpW+=tmpWidths[tpI];
                                    }
                                }
                                if(btI<cols) {
                                    btW+=lDummyTable.widths[btI];
                                }
                            }
                            totW+=tmpWidthsN[totI];
                            totI++;
                        }
                       /*if( tpI<tmpWidths.length)
                       {
                           System.arraycopy(tmpWidths, tpI, tmpWidthsN, totI, tmpWidths.length-tpI);
                           totI +=tmpWidths.length-tpI;
                       }
                       else if(btI<cols)
                       {
                           System.arraycopy(lDummyTable.widths, btI, tmpWidthsN, totI, lDummyTable.widths.length-btI);
                           totI +=lDummyTable.widths.length-btI;                                                  }*/
                        tmpWidths = new float[totI];
                        System.arraycopy(tmpWidthsN, 0, tmpWidths, 0, totI);
                        lNewMaxColumns=totI;
                    }
                                     /*if ( lDummyTable.getDimension().width > lNewMaxColumns )
                   {
                       lNewMaxColumns = lDummyTable.getDimension().width;
                       lDummyColumnWidths[j] = lDummyTable.widths; // bugfix Tony Copping
                   }*/
                }
            }
            lDummyColumnWidths[j] = tmpWidths;
            lTotalColumns += lNewMaxColumns;
            lDummyWidths [j] = lNewMaxColumns;
        }
        
        // next we'll add new rows when needed
        for (i=0; i < rows.size(); i++) {
            lNewMaxRows = 1;    // holds value in how many rows the current one will be split
            for (j=0; j < columns; j++) {
                if ( Table.class.isInstance(((Row) rows.get(i)).getCell(j)) ) {
                    isTable=true;
                    lDummyTable = (Table) ((Row) rows.get(i)).getCell(j);
                    if ( lDummyTable.getDimension().height > lNewMaxRows ) {
                        lNewMaxRows = lDummyTable.getDimension().height;
                    }
                }
            }
            lTotalRows += lNewMaxRows;
            lDummyHeights [i] = lNewMaxRows;
        }
        
        if ( (lTotalColumns != columns) || (lTotalRows != rows.size()) || isTable)    // NO ADJUSTMENT
        {
            // ** WIDTH
            // set correct width for new columns
            // divide width over new nr of columns
            // Take new max columns of internal table and work out widths for each col
            lNewWidths = new float [lTotalColumns];
            int lDummy = 0;
            for (int tel=0; tel < widths.length;tel++) {
                if ( lDummyWidths[tel] != 1) {
                    // divide
                    for (int tel2 = 0; tel2 < lDummyWidths[tel]; tel2++) {
                        // lNewWidths[lDummy] = widths[tel] / lDummyWidths[tel];
                        lNewWidths[lDummy] = widths[tel] * lDummyColumnWidths[tel][tel2] / 100f; // bugfix Tony Copping
                        lDummy++;
                    }
                }
                else {
                    lNewWidths[lDummy] = widths[tel];
                    lDummy++;
                }
            }
            
            // ** FILL OUR NEW TABLE
            // generate new table
            // set new widths
            // copy old values
            newRows = new ArrayList<Row>(lTotalRows);
            for (i = 0; i < lTotalRows; i++) {
                newRows.add(new Row(lTotalColumns));
            }
            int lDummyRow = 0, lDummyColumn = 0;        // to remember where we are in the new, larger table
            Object lDummyElement = null;
            for (i=0; i < rows.size(); i++) {
                lDummyColumn = 0;
                lNewMaxRows = 1;
                for (j=0; j < columns; j++) {
                    if ( Table.class.isInstance(((Row) rows.get(i)).getCell(j)) )       // copy values from embedded table
                    {
                        lDummyTable = (Table) ((Row) rows.get(i)).getCell(j);
                        
                        // Work out where columns in table table correspond to columns in current table
                        int colMap[] = new int[lDummyTable.widths.length+1];
                        int cb=0, ct=0;
                        
                        for( ; cb<lDummyTable.widths.length;cb++) {
                            colMap[cb] = lDummyColumn+ct;
                            
                            float wb;
                            wb = lDummyTable.widths[cb];
                            
                            float wt=0;
                            while( ct<lDummyWidths[j]) {
                                wt+=lDummyColumnWidths[j][ct++];
                                if(Math.abs(wb - wt) < 0.0001) break;
                            }
                        }
                        colMap[cb] = lDummyColumn+ct;
                        
                        // need to change this to work out how many cols to span
                        for (int k=0; k < lDummyTable.getDimension().height; k++) {
                            for (int l=0; l < lDummyTable.getDimension().width; l++) {
                                lDummyElement = lDummyTable.getElement(k,l);
                                if (lDummyElement != null) {
                                    int col=lDummyColumn+l;
                                    
                                    if ( Cell.class.isInstance(lDummyElement) ) {
                                        Cell lDummyC = (Cell)lDummyElement;
                                        // Find col to add cell in and set col span
                                        col = colMap[l];
                                        int ot = colMap[l+lDummyC.getColspan()];
                                        
                                        lDummyC.setColspan(ot-col);
                                    }
                                    
                                    newRows.get(k + lDummyRow).addElement(lDummyElement,col);  // use addElement to set reserved status ok in row
                                }
                            }
                        }
                    }
                    else        // copy others values
                    {
                        Object aElement = getElement(i,j);
                        
                        if ( Cell.class.isInstance(aElement) ) {
                            
                            // adjust spans for cell
                            ((Cell) aElement).setRowspan(((Cell) ((Row) rows.get(i)).getCell(j)).getRowspan() + lDummyHeights[i] - 1);
                            ((Cell) aElement).setColspan(((Cell) ((Row) rows.get(i)).getCell(j)).getColspan() + lDummyWidths[j] - 1);
                            
                            // most likely this cell covers a larger area because of the row/cols splits : define not-to-be-filled cells
                            placeCell(newRows,((Cell) aElement), new Point(lDummyRow,lDummyColumn));
                        }
                    }
                    lDummyColumn += lDummyWidths[j];
                }
                lDummyRow += lDummyHeights[i];
            }
            
            // Set our new matrix
            columns     = lTotalColumns;
            rows = newRows;
            this.widths = lNewWidths;
        }
    }
1077290
def build_input_partitions(cls, name='inputTablePartitions', input_name='input'):
        
        obj = cls(name)
        obj.exporter = 'get_input_partitions'
        obj.input_name = input_name
        return obj
642391
private void removePartitionResults(QueryResult queryResult, int partitionId) {
        List<QueryResultRow> rows = queryResult.getRows();
        rows.removeIf(resultRow -> getPartitionId(resultRow) == partitionId);
    }
396048
def partition(self):
        """"""
        step = int(math.ceil(self.num_tasks / float(self.partitions)))
        if self.indices == None:
            slice_ind = list(range(0, self.num_tasks, step))
            for start in slice_ind:
                yield self.__class__(self.partitions, 
                                     list(range(start, start + step)))
        else:
            slice_ind = list(range(0, len(self.indices), step))
            for start in slice_ind:
                if start + step <= len(self.indices):
                    yield self.__class__(self.partitions, 
                                         self.indices[start: start + step])
                else:
                    yield self.__class__(self.partitions, self.indices[start:])
236727
private void createPartitions(int numPartitions) {
		this.partitions.clear();
		
		ListMemorySegmentSource memSource = new ListMemorySegmentSource(this.availableMemory);
		
		for (int i = 0; i < numPartitions; i++) {
			this.partitions.add(new InMemoryPartition<T>(this.buildSideSerializer, i, memSource, this.segmentSize, pageSizeInBits));
		}
		this.compactionMemory = new InMemoryPartition<T>(this.buildSideSerializer, -1, memSource, this.segmentSize, pageSizeInBits);
	}
1339447
private void addDataSources(Table table, int hostId,
            List<Pair<Integer, Integer>> localPartitionsToSites,
            Set<Integer> partitionsInUse,
            final ExportDataProcessor processor,
            final long genId,
            boolean isCatalogUpdate)
    {
        for (Pair<Integer, Integer> partitionAndSiteId : localPartitionsToSites) {

            /*
             * IOException can occur if there is a problem
             * with the persistent aspects of the datasource storage
             */
            int partition = partitionAndSiteId.getFirst();
            int siteId = partitionAndSiteId.getSecond();
            synchronized(m_dataSourcesByPartition) {
                try {
                    Map<String, ExportDataSource> dataSourcesForPartition = m_dataSourcesByPartition.get(partition);
                    if (dataSourcesForPartition == null) {
                        dataSourcesForPartition = new HashMap<String, ExportDataSource>();
                        m_dataSourcesByPartition.put(partition, dataSourcesForPartition);
                    }
                    final String key = table.getTypeName();
                    if (!dataSourcesForPartition.containsKey(key)) {
                        ExportDataSource exportDataSource = new ExportDataSource(this,
                                processor,
                                "database",
                                key,
                                partition,
                                siteId,
                                genId,
                                table.getColumns(),
                                table.getPartitioncolumn(),
                                m_directory.getPath());
                        int migrateBatchSize = CatalogUtil.getPersistentMigrateBatchSize(key);
                        exportDataSource.setupMigrateRowsDeleter(migrateBatchSize);
                        if (exportLog.isDebugEnabled()) {
                            exportLog.debug("Creating ExportDataSource for table in catalog " + key
                                    + " partition " + partition + " site " + siteId);
                        }
                        dataSourcesForPartition.put(key, exportDataSource);
                        if (isCatalogUpdate) {
                            exportDataSource.updateCatalog(table, genId);
                        }
                    } else {
                        // Associate any existing EDS to the export client in the new processor
                        ExportDataSource eds = dataSourcesForPartition.get(key);

                        ExportClientBase client = processor.getExportClient(key);
                        if (client != null) {
                            // Associate to an existing export client
                            eds.setClient(client);
                            eds.setRunEveryWhere(client.isRunEverywhere());
                        } else {
                            // Reset to no export client
                            eds.setClient(null);
                            eds.setRunEveryWhere(false);
                        }

                        // Mark in catalog only if partition is in use
                        eds.markInCatalog(partitionsInUse.contains(partition));
                        if (isCatalogUpdate) {
                            eds.updateCatalog(table, genId);
                        }
                    }
                } catch (IOException e) {
                    VoltDB.crashLocalVoltDB(
                            "Error creating datasources for table " +
                            table.getTypeName() + " host id " + hostId, true, e);
                }
            }
        }
    }