// populateTools stores uploaded tools in provider storage
// and updates the tools metadata.
511974
func (c *BootstrapCommand) populateTools(st *state.State, env environs.BootstrapEnviron) error {
	agentConfig := c.CurrentConfig()
	dataDir := agentConfig.DataDir()

	hostSeries, err := series.HostSeries()
	if err != nil {
		return errors.Trace(err)
	}
	current := version.Binary{
		Number: jujuversion.Current,
		Arch:   arch.HostArch(),
		Series: hostSeries,
	}
	tools, err := agenttools.ReadTools(dataDir, current)
	if err != nil {
		return errors.Trace(err)
	}

	data, err := ioutil.ReadFile(filepath.Join(
		agenttools.SharedToolsDir(dataDir, current),
		"tools.tar.gz",
	))
	if err != nil {
		return errors.Trace(err)
	}

	toolstorage, err := st.ToolsStorage()
	if err != nil {
		return errors.Trace(err)
	}
	defer toolstorage.Close()

	var toolsVersions []version.Binary
	if strings.HasPrefix(tools.URL, "file://") {
		// Tools were uploaded: clone for each series of the same OS.
		os, err := series.GetOSFromSeries(tools.Version.Series)
		if err != nil {
			return errors.Trace(err)
		}
		osSeries := series.OSSupportedSeries(os)
		for _, series := range osSeries {
			toolsVersion := tools.Version
			toolsVersion.Series = series
			toolsVersions = append(toolsVersions, toolsVersion)
		}
	} else {
		// Tools were downloaded from an external source: don't clone.
		toolsVersions = []version.Binary{tools.Version}
	}

	for _, toolsVersion := range toolsVersions {
		metadata := binarystorage.Metadata{
			Version: toolsVersion.String(),
			Size:    tools.Size,
			SHA256:  tools.SHA256,
		}
		logger.Debugf("Adding agent binaries: %v", toolsVersion)
		if err := toolstorage.Add(bytes.NewReader(data), metadata); err != nil {
			return errors.Trace(err)
		}
	}
	return nil
}
136562
func (self *Record) Populate(into interface{}, collection *Collection) error {
	// special case for what is essentially copying another record into this one
	if record, ok := into.(*Record); ok {
		return self.Copy(record, collection)
	} else {
		if err := validatePtrToStructType(into); err != nil {
			return err
		}

		// if the struct we got is a zero value, and we've been given a collection,
		// use it with NewInstance
		if collection != nil {
			if typeutil.IsZero(into) {
				into = collection.NewInstance()
			}
		}

		var idFieldName string
		var fallbackIdFieldName string

		// if a collection is specified, set the fallback identity field name to what the collection
		// knows the ID field is called.  This is used for input structs that don't explicitly tag
		// a field with the ",identity" option
		if collection != nil {
			idFieldName = collection.GetIdentityFieldName()
		}

		// get the name of the identity field from the given struct
		if _, key, err := getIdentityFieldNameFromStruct(into, fallbackIdFieldName); err == nil && key != `` {
			idFieldName = key
		} else if err != nil {
			return err
		} else {
			return fmt.Errorf("Could not determine identity field name")
		}

		if data, err := self.toMap(collection, idFieldName); err == nil {
			return maputil.TaggedStructFromMap(data, into, RecordStructTag)
		} else {
			return err
		}
	}
}
635669
func (fbo *folderBlockOps) populateChainPaths(ctx context.Context,
	log logger.Logger, chains *crChains, includeCreates bool) error {
	_, err := chains.getPaths(
		ctx, fbo, log, fbo.nodeCache, includeCreates,
		fbo.config.Mode().IsTestMode())
	return err
}
1240488
func populateValuesWithKeys(keys map[string]dosa.FieldValue, values map[string]dosa.FieldValue) {
	if values != nil {
		for k, v := range keys {
			values[k] = v
		}
	}
}
887047
func (n *mutDir) populate() error {
	n.mu.Lock()
	defer n.mu.Unlock()
	ctx := context.TODO()

	// Only re-populate if we haven't done so recently.
	now := time.Now()
	if n.lastPop.Add(populateInterval).After(now) {
		return nil
	}
	n.lastPop = now

	res, err := n.fs.client.Describe(ctx, &search.DescribeRequest{
		BlobRef: n.permanode,
		Depth:   3,
	})
	if err != nil {
		Logger.Println("mutDir.paths:", err)
		return nil
	}
	db := res.Meta[n.permanode.String()]
	if db == nil {
		return errors.New("dir blobref not described")
	}

	// Find all child permanodes and stick them in n.children
	if n.children == nil {
		n.children = make(map[string]mutFileOrDir)
	}
	currentChildren := map[string]bool{}
	for k, v := range db.Permanode.Attr {
		const p = "camliPath:"
		if !strings.HasPrefix(k, p) || len(v) < 1 {
			continue
		}
		name := k[len(p):]
		childRef := v[0]
		child := res.Meta[childRef]
		if child == nil {
			Logger.Printf("child not described: %v", childRef)
			continue
		}
		if child.Permanode == nil {
			Logger.Printf("invalid child, not a permanode: %v", childRef)
			continue
		}
		if target := child.Permanode.Attr.Get("camliSymlinkTarget"); target != "" {
			// This is a symlink.
			n.maybeAddChild(name, child.Permanode, &mutFile{
				fs:        n.fs,
				permanode: blob.ParseOrZero(childRef),
				parent:    n,
				name:      name,
				symLink:   true,
				target:    target,
			})
		} else if isDir(child.Permanode) {
			// This is a directory.
			n.maybeAddChild(name, child.Permanode, &mutDir{
				fs:        n.fs,
				permanode: blob.ParseOrZero(childRef),
				parent:    n,
				name:      name,
			})
		} else if contentRef := child.Permanode.Attr.Get("camliContent"); contentRef != "" {
			// This is a file.
			content := res.Meta[contentRef]
			if content == nil {
				Logger.Printf("child content not described: %v", childRef)
				continue
			}
			if content.CamliType != "file" {
				Logger.Printf("child not a file: %v", childRef)
				continue
			}
			if content.File == nil {
				Logger.Printf("camlitype \"file\" child %v has no described File member", childRef)
				continue
			}
			n.maybeAddChild(name, child.Permanode, &mutFile{
				fs:        n.fs,
				permanode: blob.ParseOrZero(childRef),
				parent:    n,
				name:      name,
				content:   blob.ParseOrZero(contentRef),
				size:      content.File.Size,
			})
		} else {
			// unhandled type...
			continue
		}
		currentChildren[name] = true
	}
	// Remove unreferenced children
	for name, oldchild := range n.children {
		if _, ok := currentChildren[name]; !ok {
			if oldchild.eligibleToDelete() {
				delete(n.children, name)
			}
		}
	}
	return nil
}
40110
func (n *roFileVersionsDir) populate(ctx context.Context) error {
	n.mu.Lock()
	defer n.mu.Unlock()

	// Things never change here, so if we've ever populated, we're
	// populated.
	if n.children != nil {
		return nil
	}

	Logger.Printf("roFileVersionsDir.populate(%q)", n.fullPath())
	res, err := n.fs.client.GetClaims(ctx, &search.ClaimsRequest{n.permanode, "camliContent"})
	if err != nil {
		return errors.New("error while getting claims")
	}

	n.children = make(map[string]roFileOrDir)
	for _, cl := range res.Claims {
		pn, ok := blob.Parse(cl.Value)
		if !ok {
			return errors.New("invalid blobref")
		}
		res, err := n.fs.client.Describe(ctx, &search.DescribeRequest{
			BlobRef: pn, // this is camliContent
			Depth:   1,
			At:      cl.Date,
		})
		if err != nil {
			return errors.New("blobref not described")
		}
		db := res.Meta[cl.Value]
		if db == nil {
			return errors.New("blobref not described")
		}
		name := cl.Date.String()
		n.children[name] = &roFileVersion{
			fs:        n.fs,
			permanode: n.permanode,
			parent:    n,
			name:      name,
			content:   db.BlobRef,
			size:      db.File.Size,
			mtime:     cl.Date.Time(),
		}

	}
	return nil
}
1288383
func (f *ToolsFinder) FindTools(args params.FindToolsParams) (params.FindToolsResult, error) {
	result := params.FindToolsResult{}
	list, err := f.findTools(args)
	if err != nil {
		result.Error = ServerError(err)
	} else {
		result.List = list
	}
	return result, nil
}
213528
func (u *UnitUpgraderAPI) Tools(args params.Entities) (params.ToolsResults, error) {
	result := params.ToolsResults{
		Results: make([]params.ToolsResult, len(args.Entities)),
	}
	for i, entity := range args.Entities {
		result.Results[i].Error = common.ServerError(common.ErrPerm)
		tag, err := names.ParseTag(entity.Tag)
		if err != nil {
			continue
		}
		if u.authorizer.AuthOwner(tag) {
			result.Results[i] = u.getMachineTools(tag)
		}
	}
	return result, nil
}
1198266
func populateTimeSeriesData(testData TestToBuildData, testInfo *TestInfo, testTime *TestTime, job, build string, source Downloader) error {
	contents, err := getDataFromFiles(job, build, "time_series", source)
	if err != nil {
		return err
	}
	for _, data := range contents {
		// Decode the data into obj.
		var obj nodeperftype.NodeTimeSeries
		if err := json.Unmarshal(data, &obj); err != nil {
			return fmt.Errorf("failed to parse time_series data: %v\ndata=\n%v", err, string(data))
		}

		// Ignore the metrics with unsupported versions.
		if obj.Version != supportedMetricVersion {
			continue
		}

		// Populate the metadata (testInfo and testTime) with the
		// version and labels in the perf data.
		if err := populateMetadata(testInfo, testTime, obj.Labels); err != nil {
			return err
		}

		// Populate the result (testData) with the perf data.
		node := formatNodeName(obj.Labels, job)
		test := obj.Labels["test"]
		data := testData.GetDataPerBuild(job, build, test, node)
		data.Series = append(data.Series, obj)

		removeStaledBuilds(testData, job, test, node, build)
	}
	return nil
}
538861
func (p *propertySet) PopulateProposed() {

	// Reset the proposed properties
	p.proposedValues = make(map[string]uint64)
	p.clearedValues = make(map[string]uint64)

	// Gather the set of proposed stops.
	var stopping []*structs.Allocation
	for _, updates := range p.ctx.Plan().NodeUpdate {
		stopping = append(stopping, updates...)
	}
	stopping = p.filterAllocs(stopping, false)

	// Gather the proposed allocations
	var proposed []*structs.Allocation
	for _, pallocs := range p.ctx.Plan().NodeAllocation {
		proposed = append(proposed, pallocs...)
	}
	proposed = p.filterAllocs(proposed, true)

	// Get the used nodes
	both := make([]*structs.Allocation, 0, len(stopping)+len(proposed))
	both = append(both, stopping...)
	both = append(both, proposed...)
	nodes, err := p.buildNodeMap(both)
	if err != nil {
		p.errorBuilding = err
		p.logger.Error("failed to build node map", "error", err)
		return
	}

	// Populate the cleared values
	p.populateProperties(stopping, nodes, p.clearedValues)

	// Populate the proposed values
	p.populateProperties(proposed, nodes, p.proposedValues)

	// Remove any cleared value that is now being used by the proposed allocs
	for value := range p.proposedValues {
		current, ok := p.clearedValues[value]
		if !ok {
			continue
		} else if current == 0 {
			delete(p.clearedValues, value)
		} else if current > 1 {
			p.clearedValues[value]--
		}
	}
}
605120
func (m *GopenGraph) PopulateOgTags(doc *goquery.Document) {
	op := make(map[string]string)

	doc.Find("meta").Each(func(i int, s *goquery.Selection) {
		prop, _ := s.Attr("property")
		match, _ := regexp.MatchString("^og:(.+)$", prop)

		if match {
			val, _ := s.Attr("content")
			op[prop] = val
		}

	})

	m.OgAttrs = op
}
323059
func (s *Story) populateSources(ctx context.Context, st storage.Store, b literal.Builder) error {
	for _, src := range s.Sources {
		g, err := getGraphFromStore(ctx, st, src.ID)
		if err != nil {
			return err
		}
		var trps []*triple.Triple
		for _, trp := range src.Facts {
			t, err := triple.Parse(trp, b)
			if err != nil {
				return err
			}
			trps = append(trps, t)
		}
		if err := g.AddTriples(ctx, trps); err != nil {
			return err
		}
	}
	return nil
}
321357
func NewPopulator(config *common.RuntimeConfig) *Populator {
	p := &Populator{RuntimeConfig: config}
	sharedInformer := config.InformerFactory.Core().V1().PersistentVolumes()
	sharedInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			pv, ok := obj.(*v1.PersistentVolume)
			if !ok {
				glog.Errorf("Added object is not a v1.PersistentVolume type")
				return
			}
			p.handlePVUpdate(pv)
		},
		UpdateFunc: func(oldObj, newObj interface{}) {
			newPV, ok := newObj.(*v1.PersistentVolume)
			if !ok {
				glog.Errorf("Updated object is not a v1.PersistentVolume type")
				return
			}
			p.handlePVUpdate(newPV)
		},
		DeleteFunc: func(obj interface{}) {
			pv, ok := obj.(*v1.PersistentVolume)
			if !ok {
				glog.Errorf("Added object is not a v1.PersistentVolume type")
				return
			}
			p.handlePVDelete(pv)
		},
	})
	return p
}
1116938
func (task *provisionerTask) populateExcludedMachines(machineId string, startInstanceParams environs.StartInstanceParams) error {
	zonedEnv, ok := task.broker.(providercommon.ZonedEnviron)
	if !ok {
		return nil
	}
	derivedZones, err := zonedEnv.DeriveAvailabilityZones(task.cloudCallCtx, startInstanceParams)
	if err != nil {
		return errors.Trace(err)
	}
	if len(derivedZones) == 0 {
		return nil
	}
	task.machinesMutex.Lock()
	defer task.machinesMutex.Unlock()
	useZones := set.NewStrings(derivedZones...)
	for _, zoneMachines := range task.availabilityZoneMachines {
		if !useZones.Contains(zoneMachines.ZoneName) {
			zoneMachines.ExcludedMachineIds.Add(machineId)
		}
	}
	return nil
}
711602
func (c *Client) FindTools(args params.FindToolsParams) (params.FindToolsResult, error) {
	if err := c.checkCanWrite(); err != nil {
		return params.FindToolsResult{}, err
	}

	return c.api.toolsFinder.FindTools(args)
}
432974
func UnpackTools(dataDir string, tools *coretools.Tools, r io.Reader) (err error) {
	// Unpack the gzip file and compute the checksum.
	sha256hash := sha256.New()
	zr, err := gzip.NewReader(io.TeeReader(r, sha256hash))
	if err != nil {
		return err
	}
	defer zr.Close()
	f, err := ioutil.TempFile(os.TempDir(), "tools-tar")
	if err != nil {
		return err
	}
	_, err = io.Copy(f, zr)
	if err != nil {
		return err
	}
	defer os.Remove(f.Name())
	gzipSHA256 := fmt.Sprintf("%x", sha256hash.Sum(nil))
	if tools.SHA256 != gzipSHA256 {
		return fmt.Errorf("tarball sha256 mismatch, expected %s, got %s", tools.SHA256, gzipSHA256)
	}

	// Make a temporary directory in the tools directory,
	// first ensuring that the tools directory exists.
	toolsDir := path.Join(dataDir, "tools")
	err = os.MkdirAll(toolsDir, dirPerm)
	if err != nil {
		return err
	}
	dir, err := ioutil.TempDir(toolsDir, "unpacking-")
	if err != nil {
		return err
	}
	defer removeAll(dir)

	// Checksum matches, now reset the file and untar it.
	_, err = f.Seek(0, 0)
	if err != nil {
		return err
	}
	tr := tar.NewReader(f)
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		if strings.ContainsAny(hdr.Name, "/\\") {
			return fmt.Errorf("bad name %q in agent binary archive", hdr.Name)
		}
		if hdr.Typeflag != tar.TypeReg {
			return fmt.Errorf("bad file type %c in file %q in agent binary archive", hdr.Typeflag, hdr.Name)
		}
		name := path.Join(dir, hdr.Name)
		if err := writeFile(name, os.FileMode(hdr.Mode&0777), tr); err != nil {
			return errors.Annotatef(err, "tar extract %q failed", name)
		}
	}
	if err = WriteToolsMetadataData(dir, tools); err != nil {
		return err
	}

	// The tempdir is created with 0700, so we need to make it more
	// accessible for juju-run.
	err = os.Chmod(dir, dirPerm)
	if err != nil {
		return err
	}

	err = os.Rename(dir, SharedToolsDir(dataDir, tools.Version))
	// If we've failed to rename the directory, it may be because
	// the directory already exists - if ReadTools succeeds, we
	// assume all's ok.
	if err != nil {
		if _, err := ReadTools(dataDir, tools.Version); err == nil {
			return nil
		}
	}
	return err
}
293480
func (f *ToolsFinder) findTools(args params.FindToolsParams) (coretools.List, error) {
	list, err := f.findMatchingTools(args)
	if err != nil {
		return nil, err
	}
	// Rewrite the URLs so they point at the API servers. If the
	// tools are not in tools storage, then the API server will
	// download and cache them if the client requests that version.
	var fullList coretools.List
	for _, baseTools := range list {
		urls, err := f.urlGetter.ToolsURLs(baseTools.Version)
		if err != nil {
			return nil, err
		}
		for _, url := range urls {
			tools := *baseTools
			tools.URL = url
			fullList = append(fullList, &tools)
		}
	}
	return fullList, nil
}