// bulkImportStandard performs a bulk import on a standard fragment. May mutate
// its rowIDs and columnIDs arguments.
178668
func (f *fragment) bulkImportStandard(rowIDs, columnIDs []uint64, options *ImportOptions) (err error) {
	// rowSet maintains the set of rowIDs present in this import. It allows the
	// cache to be updated once per row, instead of once per bit. TODO: consider
	// sorting by rowID/columnID first and avoiding the map allocation here. (we
	// could reuse rowIDs to store the list of unique row IDs)
	rowSet := make(map[uint64]struct{})
	lastRowID := uint64(1 << 63)

	// replace columnIDs with calculated positions to avoid allocation.
	for i := 0; i < len(columnIDs); i++ {
		rowID, columnID := rowIDs[i], columnIDs[i]
		pos, err := f.pos(rowID, columnID)
		if err != nil {
			return err
		}
		columnIDs[i] = pos

		// Add row to rowSet.
		if rowID != lastRowID {
			lastRowID = rowID
			rowSet[rowID] = struct{}{}
		}
	}
	positions := columnIDs
	f.mu.Lock()
	defer f.mu.Unlock()
	if options.Clear {
		err = f.importPositions(nil, positions, rowSet)
	} else {
		err = f.importPositions(positions, nil, rowSet)
	}
	return errors.Wrap(err, "bulkImportStandard")
}
1242851
func NormalizeImportPath(path string) string {
	chunks := strings.Split(path, "/vendor/")
	return chunks[len(chunks)-1]
}
682574
func (bt *Translator) BulkAdd(field string, values [][]byte) error {
	var batchSize uint64 = 10000
	var batch uint64
	for batch*batchSize < uint64(len(values)) {
		err := bt.Db.Batch(func(tx *bolt.Tx) error {
			fib := tx.Bucket(idBucket).Bucket([]byte(field))
			fvb := tx.Bucket(valBucket).Bucket([]byte(field))

			for i := batch * batchSize; i < (batch+1)*batchSize && i < uint64(len(values)); i++ {
				idBytes := make([]byte, 8)
				binary.BigEndian.PutUint64(idBytes, i)
				valBytes := values[i]
				err := fib.Put(idBytes, valBytes)
				if err != nil {
					return errors.Wrap(err, "putting into idKey bucket")
				}
				err = fvb.Put(valBytes, idBytes)
				if err != nil {
					return errors.Wrap(err, "putting into valKey bucket")
				}
			}
			return nil
		})
		if err != nil {
			return errors.Wrap(err, "inserting batch")
		}
		batch++
	}
	return nil
}
188108
func (i *ImportRestriction) isForbidden(imp string) bool {
	importsBelowRoot := strings.HasPrefix(imp, rootPackage)
	importsBelowBase := strings.HasPrefix(imp, i.BaseDir)
	importsAllowed := false
	for _, allowed := range i.AllowedImports {
		exactlyImportsAllowed := imp == allowed
		importsBelowAllowed := strings.HasPrefix(imp, fmt.Sprintf("%s/", allowed))
		importsAllowed = importsAllowed || (importsBelowAllowed || exactlyImportsAllowed)
	}

	return importsBelowRoot && !importsBelowBase && !importsAllowed
}
505031
func importDatacenters(c context.Context, configSet config.Set, platformIds map[string]int64) error {
	cfg := &configPB.Datacenters{}
	metadata := &config.Meta{}
	if err := cfgclient.Get(c, cfgclient.AsService, configSet, datacentersFilename, textproto.Message(cfg), metadata); err != nil {
		return errors.Annotate(err, "failed to load %s", datacentersFilename).Err()
	}
	logging.Infof(c, "Found %s revision %q", datacentersFilename, metadata.Revision)

	ctx := &validation.Context{Context: c}
	ctx.SetFile(datacentersFilename)
	validateDatacentersCfg(ctx, cfg)
	if err := ctx.Finalize(); err != nil {
		return errors.Annotate(err, "invalid config").Err()
	}

	// cfgs will be a map of datacenter config filename to Datacenter.
	cfgs := make(map[string]*configPB.Datacenter, len(cfg.Datacenter))
	// datacenters will be a slice of Datacenters.
	datacenters := make([]*configPB.Datacenter, 0, len(cfg.Datacenter))
	for _, datacenterFile := range cfg.Datacenter {
		datacenter := &configPB.Datacenter{}
		if err := cfgclient.Get(c, cfgclient.AsService, configSet, datacenterFile, textproto.Message(datacenter), nil); err != nil {
			return errors.Annotate(err, "failed to load datacenter config %q", datacenterFile).Err()
		}
		cfgs[datacenterFile] = datacenter
		datacenters = append(datacenters, datacenter)
		logging.Infof(c, "Found configured datacenter %q", datacenter.Name)
	}

	validateDatacenters(ctx, cfgs)
	if err := ctx.Finalize(); err != nil {
		return errors.Annotate(err, "invalid config").Err()
	}

	datacenterIds, err := model.EnsureDatacenters(c, datacenters)
	if err != nil {
		return errors.Annotate(err, "failed to ensure datacenters").Err()
	}
	rackIds, err := model.EnsureRacks(c, datacenters, datacenterIds)
	if err != nil {
		return errors.Annotate(err, "failed to ensure racks").Err()
	}
	err = model.EnsureSwitches(c, datacenters, rackIds)
	if err != nil {
		return errors.Annotate(err, "failed to ensure switches").Err()
	}
	kvmIds, err := model.EnsureKVMs(c, datacenters, platformIds, rackIds)
	if err != nil {
		return errors.Annotate(err, "failed to ensure KVMs").Err()
	}
	// KVMs must reference the rack they exist in, while racks may optionally reference
	// a KVM that serves them. Because of this bidirectional foreign key constraint in
	// the database, we must first ensure that everything about racks except their KVM
	// matches the config, then ensure that KVMs match the config, then ensure the KVM
	// referred to by each rack matches the config.
	err = model.EnsureRackKVMs(c, datacenters, kvmIds)
	if err != nil {
		return errors.Annotate(err, "failed to ensure rack KVMs").Err()
	}
	return nil
}
142569
func newImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
	Importer = ctl.NewImportCommand(stdin, stdout, stderr)
	importCmd := &cobra.Command{
		Use:   "import",
		Short: "Bulk load data into pilosa.",
		Long: `Bulk imports one or more CSV files to a host's index and field. The data
of the CSV file are grouped by shard for the most efficient import.

The format of the CSV file is:

	ROWID,COLUMNID,[TIME]

The file should contain no headers. The TIME column is optional and can be
omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
`,
		RunE: func(cmd *cobra.Command, args []string) error {
			Importer.Paths = args
			return Importer.Run(context.Background())
		},
	}

	flags := importCmd.Flags()
	flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
	flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.")
	flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.")
	flags.BoolVar(&Importer.IndexOptions.Keys, "index-keys", false, "Specify keys=true when creating an index")
	flags.BoolVar(&Importer.FieldOptions.Keys, "field-keys", false, "Specify keys=true when creating a field")
	flags.StringVar(&Importer.FieldOptions.Type, "field-type", "", "Specify the field type when creating a field. One of: set, int, time, bool, mutex")
	flags.Int64Var(&Importer.FieldOptions.Min, "field-min", 0, "Specify the minimum for an int field on creation")
	flags.Int64Var(&Importer.FieldOptions.Max, "field-max", 0, "Specify the maximum for an int field on creation")
	flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Specify the cache type for a set field on creation. One of: none, lru, ranked")
	flags.Uint32Var(&Importer.FieldOptions.CacheSize, "field-cache-size", 50000, "Specify the cache size for a set field on creation")
	flags.Var(&Importer.FieldOptions.TimeQuantum, "field-time-quantum", "Specify the time quantum for a time field on creation. One of: D, DH, H, M, MD, MDH, Y, YM, YMD, YMDH")
	flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.")
	flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.")
	flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.")
	flags.BoolVarP(&Importer.Clear, "clear", "", false, "Clear the data provided in the import.")
	ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.SkipVerify)

	return importCmd
}
977357
func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *ImportCommand {
	return &ImportCommand{
		CmdIO:      pilosa.NewCmdIO(stdin, stdout, stderr),
		BufferSize: 10000000,
	}
}
892022
func (imp *imports) AddImportsFrom(t types.Type) {
	switch el := t.(type) {
	case *types.Basic:
	case *types.Slice:
		imp.AddImportsFrom(el.Elem())
	case *types.Pointer:
		imp.AddImportsFrom(el.Elem())
	case *types.Named:
		pkg := el.Obj().Pkg()
		if pkg == nil {
			return
		}
		if pkg.Name() == imp.currentpkg {
			return
		}
		imp.imp[cleanImportPath(pkg.Path())] = pkg.Name()
	case *types.Tuple:
		for i := 0; i < el.Len(); i++ {
			imp.AddImportsFrom(el.At(i).Type())
		}
	default:
	}
}
272039
func (c *Client) GetExportClientsByID(
	ctx context.Context, id int) ([]string, error) {

	ex, err := c.GetExportByID(ctx, id)
	if err != nil {
		return nil, err
	}
	if ex == nil {
		return nil, nil
	}
	if ex.Clients == nil {
		return nil, nil
	}
	return *ex.Clients, nil
}
654189
func (s *DescribeContinuousExportsInput) SetExportIds(v []*string) *DescribeContinuousExportsInput {
	s.ExportIds = v
	return s
}
501050
func (ce *ContainerEndpoints) AddImport(cancel <-chan struct{}, application string, bind zkservice.ImportBinding) {
	logger := plog.WithFields(log.Fields{
		"application":     application,
		"applicationglob": bind.Application,
		"purpose":         bind.Purpose,
	})
	logger.Debug("Tracking exports for endpoint")
	defer logger.Debug("Exited export tracking for endpoint")
	bind.Application = application

	for {
		select {
		case conn := <-zzk.Connect("/", zzk.GetLocalConnection):
			if conn != nil {

				logger.Debug("Received coordinator connection")

				ch := registry.TrackExports(cancel, conn, ce.opts.TenantID, application)
				for exports := range ch {
					ce.UpdateRemoteExports(bind, exports)
				}

				select {
				case <-cancel:
					return
				default:
				}
			}
		case <-cancel:
			return
		}
	}
}
739976
func NewImportPubKeyCmd(pubKey string, rescan *bool) *ImportPubKeyCmd {
	return &ImportPubKeyCmd{
		PubKey: pubKey,
		Rescan: rescan,
	}
}
1341718
func (c *rankCache) BulkAdd(id uint64, n uint64) {
	c.mu.Lock()
	defer c.mu.Unlock()
	if n < c.thresholdValue {
		return
	}

	c.entries[id] = n
}
914383
func (s *GetImportOutput) SetImportId(v string) *GetImportOutput {
	s.ImportId = &v
	return s
}
508030
func (s *CancelResizeOutput) SetImportTablesCompleted(v []*string) *CancelResizeOutput {
	s.ImportTablesCompleted = v
	return s
}
273451
func appendImports(i, a []*ImportSpec) []*ImportSpec {
	for _, v := range a {
		contains := false
		for _, att := range i {
			if att.Path == v.Path {
				contains = true
				break
			}
		}
		if contains != true {
			i = append(i, v)
		}
	}
	return i
}
635617
func pkgIDAndImportCount(c redis.Conn, path string) (id string, numImported int, err error) {
	numImported, err = redis.Int(c.Do("SCARD", "index:import:"+path))
	if err != nil {
		return
	}
	id, err = redis.String(c.Do("HGET", "ids", path))
	if err == redis.ErrNil {
		return "", 0, nil
	} else if err != nil {
		return "", 0, err
	}
	return id, numImported, nil
}