// FileUploadFromGlob allows you to create a FileUpload struct slice by just specifying a glob location on the disk
// this function will gloss over all errors in the files and only upload the files that don't return errors from the glob
168040
func FileUploadFromGlob(fileSystemGlob string) ([]FileUpload, error) {
	files, err := filepath.Glob(fileSystemGlob)

	if err != nil {
		return nil, err
	}

	if len(files) == 0 {
		return nil, errors.New("grequests: No files have been returned in the glob")
	}

	filesToUpload := make([]FileUpload, 0, len(files))

	for _, f := range files {
		if s, err := os.Stat(f); err != nil || s.IsDir() {
			continue
		}

		// ignoring error because I can stat the file
		fd, _ := os.Open(f)

		filesToUpload = append(filesToUpload, FileUpload{FileContents: fd, FileName: filepath.Base(fd.Name())})

	}

	return filesToUpload, nil

}
207958
func trimGlobs(g1, g2 Glob) (Glob, Glob, bool) {
	var l, r1, r2 int

	// Trim from the beginning until a flagged Token or a mismatch is found.
	for l = 0; l < len(g1) && l < len(g2) && g1[l].Flag() == FlagNone && g2[l].Flag() == FlagNone; l++ {
		if !Match(g1[l], g2[l]) {
			return nil, nil, false
		}
	}

	// Leave one prefix Token untrimmed to avoid empty Globs because those will break the algorithm.
	if l > 0 {
		l--
	}

	// Trim from the end until a flagged Token or a mismatch is found.
	for r1, r2 = len(g1)-1, len(g2)-1; r1 >= 0 && r1 >= l && r2 >= 0 && r2 >= l && g1[r1].Flag() == FlagNone && g2[r2].Flag() == FlagNone; r1, r2 = r1-1, r2-1 {
		if !Match(g1[r1], g2[r2]) {
			return nil, nil, false
		}
	}

	// Leave one suffix Token untrimmed to avoid empty Globs because those will break the algorithm.
	if r1 < len(g1)-1 {
		r1++
		r2++
	}

	return g1[l : r1+1], g2[l : r2+1], true
}
316761
func GetGlobArgs(args []string) ([]string, error) {

	var a []string

	for _, v := range args {
		files, err := filepath.Glob(v)
		if err != nil {
			return nil, err
		}
		a = append(a, files...)
	}

	return a, nil
}
1202578
func (s *storageImpl) FinishUpload(c context.Context, r *api.FinishUploadRequest) (resp *api.UploadOperation, err error) {
	defer func() { err = grpcutil.GRPCifyAndLogErr(c, err) }()

	if r.ForceHash != nil {
		if err := common.ValidateObjectRef(r.ForceHash, common.KnownHash); err != nil {
			return nil, errors.Annotate(err, "bad 'force_hash' field").Err()
		}
	}

	// Grab the corresponding operation and inspect its status.
	op, err := fetchOp(c, r.UploadOperationId)
	switch {
	case err != nil:
		return nil, err
	case op.Status != api.UploadStatus_UPLOADING:
		// Nothing to do if the operation is already closed or being verified.
		return op.ToProto(r.UploadOperationId), nil
	}

	// If the forced hash is provided by the (trusted) caller, we are almost done.
	// Just need to move the temp file to its final location based on this hash
	// and close the operation.
	if r.ForceHash != nil {
		mutated, err := s.finishAndForcedHash(c, op, r.ForceHash)
		if err != nil {
			return nil, err
		}
		return mutated.ToProto(r.UploadOperationId), nil
	}

	// Otherwise start the hash verification task, see verifyUploadTask below.
	mutated, err := op.Advance(c, func(c context.Context, op *upload.Operation) error {
		op.Status = api.UploadStatus_VERIFYING
		return s.tq.AddTask(c, &tq.Task{
			Payload: &tasks.VerifyUpload{UploadOperationId: op.ID},
			Title:   fmt.Sprintf("%d", op.ID),
		})
	})
	if err != nil {
		return nil, errors.Annotate(err, "failed to start the verification task").
			Tag(grpcutil.InternalTag).Err()
	}
	return mutated.ToProto(r.UploadOperationId), nil
}
1098351
func CompileGlob(pat string) (glob Glob, err error) {
	pat = toSlash(pat)
	s, err := translateGlob(pat)
	if err != nil {
		return
	}
	r, err := regexp.Compile(s)
	if err != nil {
		return
	}
	glob = Glob{pat, r}
	return
}
33005
func (c *Client) Upload(ctx context.Context, h *UploadHandle) (*PutResult, error) {
	errorf := func(msg string, arg ...interface{}) (*PutResult, error) {
		err := fmt.Errorf(msg, arg...)
		c.printf("%v", err)
		return nil, err
	}

	bodyReader, bodySize, err := h.readerAndSize()
	if err != nil {
		return nil, fmt.Errorf("client: error slurping upload handle to find its length: %v", err)
	}
	if bodySize > constants.MaxBlobSize {
		return nil, errors.New("client: body is bigger then max blob size")
	}

	c.statsMutex.Lock()
	c.stats.UploadRequests.Blobs++
	c.stats.UploadRequests.Bytes += bodySize
	c.statsMutex.Unlock()

	pr := &PutResult{BlobRef: h.BlobRef, Size: uint32(bodySize)}

	if c.sto != nil {
		// TODO: stat first so we can show skipped?
		_, err := blobserver.Receive(ctx, c.sto, h.BlobRef, bodyReader)
		if err != nil {
			return nil, err
		}
		return pr, nil
	}

	if !h.Vivify {
		if _, ok := c.haveCache.StatBlobCache(h.BlobRef); ok {
			pr.Skipped = true
			return pr, nil
		}
	}

	blobrefStr := h.BlobRef.String()

	// Pre-upload. Check whether the blob already exists on the
	// server and if not, the URL to upload it to.
	pfx, err := c.prefix()
	if err != nil {
		return nil, err
	}

	if !h.SkipStat {
		url_ := fmt.Sprintf("%s/camli/stat", pfx)
		req := c.newRequest(ctx, "POST", url_, strings.NewReader("camliversion=1&blob1="+blobrefStr))
		req.Header.Add("Content-Type", "application/x-www-form-urlencoded")

		resp, err := c.doReqGated(req)
		if err != nil {
			return errorf("stat http error: %v", err)
		}
		defer resp.Body.Close()

		if resp.StatusCode != 200 {
			return errorf("stat response had http status %d", resp.StatusCode)
		}

		stat, err := parseStatResponse(resp)
		if err != nil {
			return nil, err
		}
		for _, sbr := range stat.HaveMap {
			c.haveCache.NoteBlobExists(sbr.Ref, uint32(sbr.Size))
		}
		_, serverHasIt := stat.HaveMap[blobrefStr]
		if env.DebugUploads() {
			log.Printf("HTTP Stat(%s) = %v", blobrefStr, serverHasIt)
		}
		if !h.Vivify && serverHasIt {
			pr.Skipped = true
			if closer, ok := h.Contents.(io.Closer); ok {
				// TODO(bradfitz): I did this
				// Close-if-possible thing early on, before I
				// knew better.  Fix the callers instead, and
				// fix the docs.
				closer.Close()
			}
			c.haveCache.NoteBlobExists(h.BlobRef, uint32(bodySize))
			return pr, nil
		}
	}

	if env.DebugUploads() {
		log.Printf("Uploading: %s (%d bytes)", blobrefStr, bodySize)
	}

	pipeReader, pipeWriter := io.Pipe()
	multipartWriter := multipart.NewWriter(pipeWriter)

	copyResult := make(chan error, 1)
	go func() {
		defer pipeWriter.Close()
		part, err := multipartWriter.CreateFormFile(blobrefStr, blobrefStr)
		if err != nil {
			copyResult <- err
			return
		}
		_, err = io.Copy(part, bodyReader)
		if err == nil {
			err = multipartWriter.Close()
		}
		copyResult <- err
	}()

	uploadURL := fmt.Sprintf("%s/camli/upload", pfx)
	req := c.newRequest(ctx, "POST", uploadURL)
	req.Header.Set("Content-Type", multipartWriter.FormDataContentType())
	if h.Vivify {
		req.Header.Add("X-Camlistore-Vivify", "1")
	}
	req.Body = ioutil.NopCloser(pipeReader)
	req.ContentLength = getMultipartOverhead() + bodySize + int64(len(blobrefStr))*2
	resp, err := c.doReqGated(req)
	if err != nil {
		return errorf("upload http error: %v", err)
	}
	defer resp.Body.Close()

	// check error from earlier copy
	if err := <-copyResult; err != nil {
		return errorf("failed to copy contents into multipart writer: %v", err)
	}

	// The only valid HTTP responses are 200 and 303.
	if resp.StatusCode != 200 && resp.StatusCode != 303 {
		return errorf("invalid http response %d in upload response", resp.StatusCode)
	}

	if resp.StatusCode == 303 {
		otherLocation := resp.Header.Get("Location")
		if otherLocation == "" {
			return errorf("303 without a Location")
		}
		baseURL, _ := url.Parse(uploadURL)
		absURL, err := baseURL.Parse(otherLocation)
		if err != nil {
			return errorf("303 Location URL relative resolve error: %v", err)
		}
		otherLocation = absURL.String()
		resp, err = http.Get(otherLocation)
		if err != nil {
			return errorf("error following 303 redirect after upload: %v", err)
		}
	}

	var ures protocol.UploadResponse
	if err := httputil.DecodeJSON(resp, &ures); err != nil {
		return errorf("error in upload response: %v", err)
	}

	if ures.ErrorText != "" {
		c.printf("Blob server reports error: %s", ures.ErrorText)
	}

	expectedSize := uint32(bodySize)

	for _, sb := range ures.Received {
		if sb.Ref != h.BlobRef {
			continue
		}
		if sb.Size != expectedSize {
			return errorf("Server got blob %v, but reports wrong length (%v; we sent %d)",
				sb.Ref, sb.Size, expectedSize)
		}
		c.statsMutex.Lock()
		c.stats.Uploads.Blobs++
		c.stats.Uploads.Bytes += bodySize
		c.statsMutex.Unlock()
		if pr.Size <= 0 {
			pr.Size = sb.Size
		}
		c.haveCache.NoteBlobExists(pr.BlobRef, pr.Size)
		return pr, nil
	}

	return nil, errors.New("server didn't receive blob")
}
267890
func (c *Client) glob(dir, pattern string, matches []string) (m []string, e error) {
	m = matches
	fi, err := c.Stat(dir)
	if err != nil {
		return
	}
	if !fi.IsDir() {
		return
	}
	names, err := c.ReadDir(dir)
	if err != nil {
		return
	}
	//sort.Strings(names)

	for _, n := range names {
		matched, err := Match(pattern, n.Name())
		if err != nil {
			return m, err
		}
		if matched {
			m = append(m, Join(dir, n.Name()))
		}
	}
	return
}
924743
func (ctx *Context) PushGlobalGoFunction(name string, f interface{}) (int, error) {
	return ctx.Context.PushGlobalGoFunction(name, ctx.wrapFunction(f))
}
291869
func NewGlobSet(globs []Glob) (GlobSet, error) {
	set := make(globSetImpl, len(globs))
	for i, glob := range globs {
		set[i] = glob
	}
	return set, nil
}
1178638
func (f FileInfo) GoPluginOutputFilename() string {
	name := f.Name
	ext := path.Ext(name)
	if ext == ".proto" || ext == ".protodevel" {
		name = name[0 : len(name)-len(ext)]
	}
	return name + ".pb.go"
}
415702
func Glob(pattern string) (matches []string, err error) {
	if !hasMeta(pattern) {
		if _, err = os.Lstat(pattern); err != nil {
			return nil, nil
		}
		return []string{pattern}, nil
	}
	dir, file := filepath.Split(pattern)
	switch dir {
	case "":
		dir = "."
	case string(filepath.Separator):
		// nothing
	default:
		dir = dir[0 : len(dir)-1] // chop off trailing separator
	}
	if !hasMeta(dir) {
		return glober(dir, file, nil)
	}
	var m []string
	m, err = Glob(dir)
	if err != nil {
		return
	}
	for _, d := range m {
		matches, err = glober(d, file, matches)
		if err != nil {
			return
		}
	}
	return
}
516413
func (s *S3Uploader) UploadToS3(fPath string, buf *bytes.Buffer) error {
	fmt.Fprintf(s.out, "Preparing %q for upload to s3 bucket %q...", filepath.Base(fPath), s.s3url.String())

	req, err := s.buildS3Request(filepath.Base(fPath), buf)
	if err != nil {
		fmt.Fprintln(s.out, " error")
		return err
	}
	fmt.Fprintln(s.out, " done")

	fmt.Fprint(s.out, "Uploading...")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		fmt.Fprintln(s.out, " error")
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		fmt.Fprintln(s.out, " error")
		if resp.StatusCode == 403 {
			fmt.Fprintln(s.out, "Encountered authorization error! Are your keys set correctly?")
		}
		errMsg := fmt.Sprintf("received code %d, should have received %d", resp.StatusCode, http.StatusOK)
		b, err := ioutil.ReadAll(resp.Body)
		if err == nil {
			errMsg = fmt.Sprintf("%s\n\nReceived S3 Response: %q", errMsg, string(b))
		}
		return fmt.Errorf(errMsg)
	}
	fmt.Fprintln(s.out, " done")
	return nil
}
986524
func NewGFileForPath(filename string) *GFile {
	ptrFilename := C.CString(filename)
	defer cfree(ptrFilename)

	return &GFile{C.g_file_new_for_path(ptrFilename)}
}
1229875
func (t *tOps) createFrom(src iterator.Iterator) (f *tFile, n int, err error) {
	w, err := t.create()
	if err != nil {
		return
	}

	defer func() {
		if err != nil {
			w.drop()
		}
	}()

	for src.Next() {
		err = w.append(src.Key(), src.Value())
		if err != nil {
			return
		}
	}
	err = src.Error()
	if err != nil {
		return
	}

	n = w.tw.EntriesLen()
	f, err = w.finish()
	return
}
627821
func (up *Uploader) NewTreeUpload(dir string) *TreeUpload {
	tu := up.NewRootlessTreeUpload()
	tu.rootless = false
	tu.base = dir
	return tu
}
695502
func (request *Request) Upload(filename string, filetype string) (response *Response, err error) {
	postRequest, err := request.buildPost(uploadEndpoint, filename, filetype)
	if err != nil {
		return nil, err
	}
	return sendPost(postRequest)
}
891373
func NewGlobOption(opts ...GlogOptFunc) *GlobOption {
	o := &GlobOption{flags: GlobStdFlags}
	for _, opt := range opts {
		opt(o)
	}
	return o
}