Set sample points from the population. Should be a RDD
348861
def setSample(self, sample):
        """"""
        if not isinstance(sample, RDD):
            raise TypeError("samples should be a RDD, received %s" % type(sample))
        self._sample = sample
1165791
def initialize_pop(self):
        """"""
        self.toolbox.register("individual", self.generate)
        self.toolbox.register("population", tools.initRepeat,
                              list, self.toolbox.individual)
        self.population = self.toolbox.population(n=self._params['popsize'])
        if self._params['neighbours']:
            for i in range(len(self.population)):
                self.population[i].ident = i
                self.population[i].neighbours = list(
                    set(
                        [(i - x) % len(self.population)
                         for x in range(1, self._params['neighbours'] + 1)] +
                        [(i + x) % len(self.population)
                         for x in range(1, self._params['neighbours'] + 1)]
                    ))
        self.assign_fitnesses(self.population)
796283
def _transform(self, dataset):
    
    spark = SparkSession.builder.getOrCreate()

    # set a deterministic order for input/output columns (lexicographic by key)
    input_cols = [col for col, tensor in sorted(self.getInputMapping().items())]      # input col => input tensor
    output_cols = [col for tensor, col in sorted(self.getOutputMapping().items())]    # output tensor => output col

    # run single-node inferencing on each executor
    logging.info("input_cols: {}".format(input_cols))
    logging.info("output_cols: {}".format(output_cols))

    # merge args + params
    logging.info("===== 1. inference args: {0}".format(self.args))
    logging.info("===== 2. inference params: {0}".format(self._paramMap))
    local_args = self.merge_args_params()
    logging.info("===== 3. inference args + params: {0}".format(local_args))

    tf_args = self.args.argv if self.args.argv else local_args
    rdd_out = dataset.select(input_cols).rdd.mapPartitions(lambda it: _run_model(it, local_args, tf_args))

    # convert to a DataFrame-friendly format
    rows_out = rdd_out.map(lambda x: Row(*x))
    return spark.createDataFrame(rows_out, output_cols)
835894
def sample(self, nsamples=100, seed=None):
        
        if nsamples < 1:
            raise ValueError("Number of samples must be larger than 0, got '%g'" % nsamples)

        if seed is None:
            seed = random.randint(0, 2 ** 32)

        if self.mode == 'spark':
            result = asarray(self.values.tordd().values().takeSample(False, nsamples, seed))

        else:
            inds = [int(k) for k in random.rand(nsamples) * self.shape[0]]
            result = asarray([self.values[i] for i in inds])

        return self._constructor(result)
4897
def samples(self):
        """"""
        names = self.series.dimensions
        for n, offset in enumerate(self.series.offsets):
            dt = datetime.timedelta(microseconds=offset * 1000)
            d = {"ts": self.ts + dt}
            for name in names:
                d[name] = getattr(self.series, name)[n]
            yield d
604647
def collect(self):
        """"""
        local_df = self._schema_rdd.toPandas()
        correct_idx_df = _update_index_on_df(local_df, self._index_names)
        return correct_idx_df
67058
def generate_dataset(number_items=1000):
    
    data = []
    names = get_names()
    totalnames = len(names)
    #init random seeder
    random.seed()
    #calculate items
#    names = random.sample(names, number_items)
    for i in range(number_items):
        data.append({"name":names[random.randint(0,totalnames-1)],
                     "age":random.randint(1,100),
                     "description":li_words(50, False)})
    return data
600763
def randomprune(self,n):
        """"""
        self.data = random.sample(self.data, n)
287906
def plot(self,init=None):
		""""""

		if not init:
			init=self
			init.plotstats={}
			init.unitfeats=self._getFeatValDict()

			sels=[]
			for k,v in sorted(init.unitfeats.items()):
				for kk,vv in v.items():
					sels.append((k,kk))

			conditions={'x':[],'y':[]}
			targets={'x':[],'y':[]}
			pkey={'x':[],'y':[]}

			print str(0)+"\t[no condition]"
			for selnum in range(len(sels)):
				print str(selnum+1)+"\t"+str(sels[selnum][0])+" = "+str(sels[selnum][1])
			print

			stepnum=0
			for a in sorted(conditions.keys()):
				stepnum+=1
				sel=raw_input(">> [step "+str(stepnum)+"/4] ["+a+" coord] [conditions of population] please type in the number (or numbers separated by commas)\n\tof the conditions determining the total population from which the percentage of the sample is taken:\n").strip()
				for x in sel.split(","):
					try:
						conditions[a].append(sels[int(x)-1])
						pkey[a]+=["of_"+str(sels[int(x)-1][0])+"_is_"+str(sels[int(x)-1][1])]
					except:
						pass

				stepnum+=1
				sel=raw_input(">> [step "+str(stepnum)+"/4] ["+a+" coord] [conditions of sample] please type in the number (or numbers separated by commas)\n\tof the conditions determining the sample:\n").strip()
				for x in sel.split(","):
					try:
						targets[a].append(sels[int(x)-1])
						pkey[a]+=[str(sels[int(x)-1][0])+"_is_"+str(sels[int(x)-1][1])]
					except:
						pass


			# print ">> POPULATION:"
			# print conditions
			#
			# print ">> SAMPLE:"
			# print targets

			init.population=conditions
			init.sample=targets

			init.pkey="X_"+"-".join(pkey['x'])+"."+"Y_"+"-".join(pkey['y'])
			print ">> plotting: "+ init.pkey


		if not hasattr(self,'bestParses'):
			for child in self.children:
				child.plot(init)
		else:
			posdict={}
			minparselen = min([len(parse.positions) for parse in self.bestParses()])
			maxparselen = max([len(parse.positions) for parse in self.bestParses()])
			#if not hasattr(self,'minparselen'): return None
			for posnum in range(maxparselen):
				posdict[posnum]={'x':[],'y':[]}
				for parse in self.bestParses():
					try:
						posfeats=parse.positions[posnum].posfeats()
					except IndexError:
						# there is no position number `posnum` in this parse `parse`
						continue

					for a in ['x','y']:
						conds=init.population[a]
						targs=init.sample[a]

						condsHold=True
						if len(conds):
							for cond in conds:
								condK=cond[0]
								condV=cond[1]

								try:
									if posfeats[condK]==condV:
										continue
								except:
									condsHold=False
									break
								condsHold=False
								break

						if condsHold:
							targsHold=True
							for targ in targs:
								targK=targ[0]
								targV=targ[1]

								try:
									if posfeats[targK]==targV:
										continue
								except:
									targsHold=False
									break
								targsHold=False
								break

							if targsHold:
								posdict[posnum][a].append(1)
							else:
								posdict[posnum][a].append(0)

			for a in ['x','y']:
				if not posdict[posnum][a]:
					print "<< not enough data: position number ("+str(posnum)+") empty on dimension ["+str(a)+"]"
					return None

			init.plotstats[self.getName()]=posdict
			if (self!=init): return None

		totalstrs=[]
		totaltsvs=[]
		for textname,posdict in sorted(init.plotstats.items()):

			tsv="posnum\tnumobs\tx_mean\ty_mean\tx_std\ty_std\n"
			xs=[]
			ys=[]
			for posnum,xydict in posdict.items():
				x_avg,x_std=mean_stdev(xydict['x'])
				y_avg,y_std=mean_stdev(xydict['y'])

				assert len(xydict['x'])==len(xydict['y'])

				xs.append(x_avg)
				ys.append(y_avg)
				tsv+="\t".join(str(bb) for bb in [(posnum+1),len(xydict['x']),x_avg,y_avg,x_std,y_std])+"\n"


			ccmsg=""
			cc=None
			p=None
			try:
				from statlib import stats
				(cc,p)=stats.pearsonr(xs,ys)

				aa=makeminlength("    correlation coefficient: ",int(being.linelen/1.4))+str(cc)
				bb=makeminlength("    p-value: ",int(being.linelen/1.4))+str(p)

				tsv+=aa.strip().replace(":",":\t")+"\n"
				tsv+=bb.strip().replace(":",":\t")+"\n"

				for l in tsv.split("\n"):
					totaltsvs.append(textname+"_"+l)

				ccmsg+=aa+"\n"+bb+"\n"

			except:
				pass

			writeToFile(textname,init.pkey,tsv,extension="tsv")
			totaltsvs.append(tsv)

			"""
			try:
				strtowrite=self.makeBubbleChart(posdict,".".join([textname,init.pkey]),(cc,p))
				totalstrs+=[strtowrite]
				writeToFile(textname,init.pkey,self._getBubbleHeader()+strtowrite+self._getBubbleFooter(),extension="htm")
			except:
				pass
			"""

			if ccmsg:
				print ccmsg

		if not self.classname()=="Corpus": return None

		"""
		writeToFile(self.getName(),
			init.pkey,
			self._getBubbleHeader()+"\n<br/><br/><br/><br/><br/><br/><br/><br/>\n".join(totalstrs)+self._getBubbleFooter(),
			iscorpus=True,
			extension="htm")
		"""

		writeToFile(self.getName(),init.pkey,"\n\n\n\n".join(totaltsvs),iscorpus=True,extension="tsv")
579424
def _setDatasets(self, metadata):
        """"""
        # Pull the lot into a list
        cols = list(self.getCSVReader(metadata))

        # sort the cols by view name and ordinal (they come sorted, but lets be sure)
        cols.sort(key=lambda x: [x['viewname'], int(x['ordinal'])])

        # Now group them by their viewname
        for key, grp in groupby(cols, key=lambda x: x["viewname"]):
            self.datasets[key] = list(grp)
157967
def sample(records, k, random_seed=None):
    

    if random_seed is not None:
        random.seed(random_seed)

    result = []
    for i, record in enumerate(records):
        if len(result) < k:
            result.append(record)
        else:
            r = random.randint(0, i)
            if r < k:
                result[r] = record
    return result
1320133
def _sample(probabilities, population_size):
    """"""
    population = []
    for _ in range(population_size):
        solution = []
        for probability in probabilities:
            # probability of 1.0: always 1
            # probability of 0.0: always 0
            if random.uniform(0.0, 1.0) < probability:
                solution.append(1)
            else:
                solution.append(0)
        population.append(solution)
    return population
969810
function () {
      for (var r = 0; r < self.rowCount; r++) {
        for (var c = 0; c < self.colCount; c++) {
          datamap.set(r, datamap.colToProp(c), '');
        }
      }
    }
1057310
def representative_sample(X, num_samples, save=False):
    
    X = X.values if hasattr(X, 'values') else np.array(X)
    N, M = X.shape
    rownums = np.arange(N)
    np.random.shuffle(rownums)

    idx = AnnoyIndex(M)
    for i, row in enumerate(X):
        idx.add_item(i, row)
    idx.build(int(np.log2(N)) + 1)

    if save:
        if isinstance(save, basestring):
            idxfilename = save
        else:
            idxfile = tempfile.NamedTemporaryFile(delete=False)
            idxfile.close()
            idxfilename = idxfile.name
        idx.save(idxfilename)
        idx = AnnoyIndex(M)
        idx.load(idxfile.name)

    samples = -1 * np.ones(shape=(num_samples,), dtype=int)
    samples[0] = rownums[0]
    # FIXME: some integer determined by N and num_samples and distribution
    j, num_nns = 0, min(1000, int(num_samples / 2. + 1))
    for i in rownums:
        if i in samples:
            continue
        nns = idx.get_nns_by_item(i, num_nns)
        # FIXME: pick vector furthest from past K (K > 1) points or outside of a hypercube
        #        (sized to uniformly fill the space) around the last sample
        samples[j + 1] = np.setdiff1d(nns, samples)[-1]
        if len(num_nns) < num_samples / 3.:
            num_nns = min(N, 1.3 * num_nns)
        j += 1
    return samples
826834
def create_population(self, popsize=1000, min_depth=2,
                          max_depth=4,
                          X=None):
        ""
        import itertools
        args = [x for x in itertools.product(range(min_depth,
                                                   max_depth+1),
                                             [True, False])]
        index = 0
        output = []
        while len(output) < popsize:
            depth, full = args[index]
            index += 1
            if index >= len(args):
                index = 0
            if full:
                ind = self.create_random_ind_full(depth=depth)
            else:
                ind = self.create_random_ind_grow(depth=depth)
            flag = True
            if X is not None:
                x = Individual(ind)
                x.decision_function(X)
                flag = x.individual[0].isfinite()
            l_vars = (flag, len(output), full, depth, len(ind))
            l_str = " flag: %s len(output): %s full: %s depth: %s len(ind): %s"
            self._logger.debug(l_str % l_vars)
            if flag:
                output.append(ind)
        return output
128118
def make_data(n):
    """"""
    V = range(1,n+1)
    x = dict([(i, random.random()) for i in V])
    y = dict([(i, random.random()) for i in V])
    c, q = {}, {}
    Q = 100
    for i in V:
        q[i] = random.randint(10, 20)
        for j in V:
            if j > i:
                c[i, j] = distance(x[i], y[i], x[j], y[j])
    return V, c, q, Q
280405
def initialize_pop(self):
        """"""
        self.population = self.toolbox.swarm(n=self._params['popsize'])
        if self._params['neighbours']:
            for i in range(len(self.population)):
                self.population[i].ident = i
                self.population[i].neighbours = list(
                    set(
                        [(i - x) % len(self.population)
                         for x in range(1, self._params['neighbours'] + 1)] +
                        [i] +
                        [(i + x) % len(self.population)
                         for x in range(1, self._params['neighbours'] + 1)]
                    ))
        else:
            for i in range(len(self.population)):
                self.population[i].ident = i
                self.population[i].neighbours = [
                    x for x in range(len(self.population))]
        self.assign_fitnesses(self.population)
        for part in self.population:
            part.best = creator.Particle(part)
            part.best.fitness.values = part.fitness.values