Pod spec to be used to create pods for project: tensorboard, notebooks.
2624
def get_project_pod_spec(volume_mounts,
                         volumes,
                         image,
                         command,
                         args,
                         ports,
                         env_vars=None,
                         env_from=None,
                         container_name=None,
                         resources=None,
                         node_selector=None,
                         affinity=None,
                         tolerations=None,
                         image_pull_policy=None,
                         restart_policy=None,
                         service_account_name=None):
    """"""
    volume_mounts = to_list(volume_mounts, check_none=True)
    volumes = to_list(volumes, check_none=True)

    gpu_volume_mounts, gpu_volumes = get_gpu_volumes_def(resources)
    volume_mounts += gpu_volume_mounts
    volumes += gpu_volumes

    ports = [client.V1ContainerPort(container_port=port) for port in ports]

    pod_container = get_pod_container(
        volume_mounts=volume_mounts,
        image=image,
        command=command,
        args=args,
        ports=ports,
        env_vars=env_vars,
        env_from=env_from,
        container_name=container_name,
        resources=resources,
        image_pull_policy=image_pull_policy)
    containers = [pod_container]

    if service_account_name and not conf.get('K8S_RBAC_ENABLED'):
        service_account_name = None

    return client.V1PodSpec(restart_policy=restart_policy,
                            security_context=get_security_context(),
                            service_account_name=service_account_name,
                            containers=containers,
                            volumes=volumes,
                            node_selector=node_selector,
                            affinity=affinity,
                            tolerations=tolerations)
382614
def specification_path(name, version)
      raise ArgumentError, 'No name' unless name
      raise ArgumentError, 'No version' unless version
      
      relative_podspec = pod_path_partial(name).join("#{version}/#{name}.podspec.json")
      download_file(relative_podspec)
      pod_path(name).join("#{version}/#{name}.podspec.json")
    end
609696
def parse_plugin_metadata(content):
  
  if not isinstance(content, bytes):
    raise TypeError('Content type must be bytes')
  result = plugin_data_pb2.PrCurvePluginData.FromString(content)
  if result.version == 0:
    return result
  else:
    logger.warn(
        'Unknown metadata version: %s. The latest version known to '
        'this build of TensorBoard is %s; perhaps a newer build is '
        'available?', result.version, PROTO_VERSION)
    return result
1260351
def get_pod_container(self,
                          volume_mounts,
                          persistence_outputs=None,
                          persistence_data=None,
                          outputs_refs_jobs=None,
                          outputs_refs_experiments=None,
                          secret_refs=None,
                          configmap_refs=None,
                          env_vars=None,
                          command=None,
                          args=None,
                          resources=None,
                          ports=None,
                          ephemeral_token=None):
        """"""
        self._pod_container_checks()

        # Env vars preparations
        env_vars = to_list(env_vars, check_none=True)
        env_vars += self._get_container_pod_env_vars(
            persistence_outputs=persistence_outputs,
            persistence_data=persistence_data,
            outputs_refs_jobs=outputs_refs_jobs,
            outputs_refs_experiments=outputs_refs_experiments,
            ephemeral_token=ephemeral_token
        )
        env_vars += get_resources_env_vars(resources=resources)

        # Env from configmap and secret refs
        env_from = get_pod_env_from(secret_refs=secret_refs, configmap_refs=configmap_refs)

        def get_ports():
            _ports = to_list(ports) if ports else []
            return [client.V1ContainerPort(container_port=port) for port in _ports] or None

        return client.V1Container(name=self.job_container_name,
                                  image=self.job_docker_image,
                                  command=command,
                                  args=args,
                                  ports=get_ports(),
                                  env=env_vars,
                                  env_from=env_from,
                                  resources=get_resources(resources),
                                  volume_mounts=volume_mounts)
382731
def export_module_spec(spec, path, checkpoint_path, name_transform_fn):
  """"""
  with tf.Graph().as_default():
    m = Module(spec)
    assign_map = {
        name_transform_fn(name): value for name, value in m.variable_map.items()
    }
    tf_v1.train.init_from_checkpoint(checkpoint_path, assign_map)
    init_op = tf_v1.initializers.global_variables()
    with tf_v1.Session() as session:
      session.run(init_op)
      m.export(path, session)
721376
def specification(cls, *context_specs):
		
		import_data = []
		for name in context_specs:
			import_data.append((name, None))
		return cls.import_context(import_data)
752871
def load_nb(cls, inline=True):
        
        LOAD_MIME_TYPE = bokeh.io.notebook.LOAD_MIME_TYPE
        bokeh.io.notebook.LOAD_MIME_TYPE = MIME_TYPES['jlab-hv-load']
        load_notebook(hide_banner=True, resources=INLINE if inline else CDN)
        bokeh.io.notebook.LOAD_MIME_TYPE = LOAD_MIME_TYPE
        bokeh.io.notebook.curstate().output_notebook()
1089185
def get_task_pod_spec(self,
                          volume_mounts,
                          volumes,
                          resource_name,
                          persistence_outputs=None,
                          persistence_data=None,
                          outputs_refs_jobs=None,
                          outputs_refs_experiments=None,
                          env_vars=None,
                          command=None,
                          args=None,
                          init_command=None,
                          init_args=None,
                          init_env_vars=None,
                          resources=None,
                          ports=None,
                          secret_refs=None,
                          configmap_refs=None,
                          ephemeral_token=None,
                          node_selector=None,
                          affinity=None,
                          tolerations=None,
                          sidecar_context_mounts=None,
                          init_context_mounts=None,
                          restart_policy=None):
        """"""
        sidecar_context_mounts = to_list(sidecar_context_mounts, check_none=True)
        init_context_mounts = to_list(init_context_mounts, check_none=True)
        volume_mounts = to_list(volume_mounts, check_none=True)
        volumes = to_list(volumes, check_none=True)

        gpu_volume_mounts, gpu_volumes = get_gpu_volumes_def(resources)
        volume_mounts += gpu_volume_mounts
        volumes += gpu_volumes

        pod_container = self.get_pod_container(volume_mounts=volume_mounts,
                                               persistence_outputs=persistence_outputs,
                                               persistence_data=persistence_data,
                                               outputs_refs_jobs=outputs_refs_jobs,
                                               outputs_refs_experiments=outputs_refs_experiments,
                                               secret_refs=secret_refs,
                                               configmap_refs=configmap_refs,
                                               resources=resources,
                                               env_vars=env_vars,
                                               command=command,
                                               args=args,
                                               ports=ports,
                                               ephemeral_token=ephemeral_token)

        containers = [pod_container]
        if self.use_sidecar:
            sidecar_volume_mounts = self.get_sidecar_volume_mounts(
                persistence_outputs=persistence_outputs,
                persistence_data=persistence_data,
                context_mounts=sidecar_context_mounts)
            sidecar_container = self.get_sidecar_container(volume_mounts=sidecar_volume_mounts)
            containers.append(sidecar_container)

        init_container = self.get_init_container(init_command=init_command,
                                                 init_args=init_args,
                                                 env_vars=init_env_vars,
                                                 context_mounts=init_context_mounts,
                                                 persistence_outputs=persistence_outputs,
                                                 persistence_data=persistence_data)
        init_containers = to_list(init_container, check_none=True)

        node_selector = self._get_node_selector(node_selector=node_selector)
        affinity = self._get_affinity(affinity=affinity)
        tolerations = self._get_tolerations(tolerations=tolerations)
        service_account_name = self._get_service_account_name()
        return client.V1PodSpec(
            security_context=get_security_context() if self.use_security_context else None,
            restart_policy=restart_policy,
            service_account_name=service_account_name,
            init_containers=init_containers,
            containers=containers,
            volumes=volumes,
            node_selector=node_selector,
            tolerations=tolerations,
            affinity=affinity)
116568
def create(model_config, batch_size, vectors=None):
    
    path = model_config.data_dir('imdb')

    text_field = data.Field(lower=True, tokenize='spacy', batch_first=True)
    label_field = data.LabelField(is_target=True)

    train_source, test_source = IMDBCached.splits(
        root=path,
        text_field=text_field,
        label_field=label_field
    )

    text_field.build_vocab(train_source, max_size=25_000, vectors=vectors)
    label_field.build_vocab(train_source)

    train_iterator, test_iterator = data.BucketIterator.splits(
        (train_source, test_source),
        batch_size=batch_size,
        device=model_config.torch_device(),
        shuffle=True
    )

    return TextData(
        train_source, test_source, train_iterator, test_iterator, text_field, label_field
    )
442657
def _report_container_spec_metrics(self, pod_list, instance_tags):
        """"""
        for pod in pod_list['items']:
            pod_name = pod.get('metadata', {}).get('name')
            pod_phase = pod.get('status', {}).get('phase')
            if self._should_ignore_pod(pod_name, pod_phase):
                continue

            for ctr in pod['spec']['containers']:
                if not ctr.get('resources'):
                    continue

                c_name = ctr.get('name', '')
                cid = None
                for ctr_status in pod['status'].get('containerStatuses', []):
                    if ctr_status.get('name') == c_name:
                        # it is already prefixed with 'runtime://'
                        cid = ctr_status.get('containerID')
                        break
                if not cid:
                    continue

                pod_uid = pod.get('metadata', {}).get('uid')
                if self.pod_list_utils.is_excluded(cid, pod_uid):
                    continue

                tags = tagger.tag('%s' % cid, tagger.HIGH) + instance_tags

                try:
                    for resource, value_str in iteritems(ctr.get('resources', {}).get('requests', {})):
                        value = self.parse_quantity(value_str)
                        self.gauge('{}.{}.requests'.format(self.NAMESPACE, resource), value, tags)
                except (KeyError, AttributeError) as e:
                    self.log.debug("Unable to retrieve container requests for %s: %s", c_name, e)

                try:
                    for resource, value_str in iteritems(ctr.get('resources', {}).get('limits', {})):
                        value = self.parse_quantity(value_str)
                        self.gauge('{}.{}.limits'.format(self.NAMESPACE, resource), value, tags)
                except (KeyError, AttributeError) as e:
                    self.log.debug("Unable to retrieve container limits for %s: %s", c_name, e)
776762
def list_pods(self, namespace=None):
        

        if namespace:
            return [Pod(name=p.metadata.name, namespace=namespace, spec=p.spec)
                    for p in self.core_api.list_namespaced_pod(namespace, watch=False).items]

        return [Pod(name=p.metadata.name, namespace=p.metadata.namespace, spec=p.spec)
                for p in self.core_api.list_pod_for_all_namespaces(watch=False).items]
778699
def custom_build(inputs, is_training, keep_prob):
  """"""
  outputs = snt.Conv2D(output_channels=32, kernel_shape=4, stride=2)(inputs)
  outputs = snt.BatchNorm()(outputs, is_training=is_training)
  outputs = tf.nn.relu(outputs)
  outputs = snt.Conv2D(output_channels=64, kernel_shape=4, stride=2)(outputs)
  outputs = snt.BatchNorm()(outputs, is_training=is_training)
  outputs = tf.nn.relu(outputs)
  outputs = snt.BatchFlatten()(outputs)
  outputs = tf.nn.dropout(outputs, keep_prob=keep_prob)
  outputs = snt.Linear(output_size=10)(outputs)
  return outputs
57123
def _get_pod_by_metric_label(self, labels):
        
        pod_uid = self._get_pod_uid(labels)
        return get_pod_by_uid(pod_uid, self.pod_list)
1278374
def get_pod_manifest(self):
        
        if callable(self.uid):
            uid = yield gen.maybe_future(self.uid(self))
        else:
            uid = self.uid

        if callable(self.gid):
            gid = yield gen.maybe_future(self.gid(self))
        else:
            gid = self.gid

        if callable(self.fs_gid):
            fs_gid = yield gen.maybe_future(self.fs_gid(self))
        else:
            fs_gid = self.fs_gid

        if callable(self.supplemental_gids):
            supplemental_gids = yield gen.maybe_future(self.supplemental_gids(self))
        else:
            supplemental_gids = self.supplemental_gids

        if self.cmd:
            real_cmd = self.cmd + self.get_args()
        else:
            real_cmd = None

        labels = self._build_pod_labels(self._expand_all(self.extra_labels))
        annotations = self._build_common_annotations(self._expand_all(self.extra_annotations))

        return make_pod(
            name=self.pod_name,
            cmd=real_cmd,
            port=self.port,
            image=self.image,
            image_pull_policy=self.image_pull_policy,
            image_pull_secret=self.image_pull_secrets,
            node_selector=self.node_selector,
            run_as_uid=uid,
            run_as_gid=gid,
            fs_gid=fs_gid,
            supplemental_gids=supplemental_gids,
            run_privileged=self.privileged,
            env=self.get_env(),
            volumes=self._expand_all(self.volumes),
            volume_mounts=self._expand_all(self.volume_mounts),
            working_dir=self.working_dir,
            labels=labels,
            annotations=annotations,
            cpu_limit=self.cpu_limit,
            cpu_guarantee=self.cpu_guarantee,
            mem_limit=self.mem_limit,
            mem_guarantee=self.mem_guarantee,
            extra_resource_limits=self.extra_resource_limits,
            extra_resource_guarantees=self.extra_resource_guarantees,
            lifecycle_hooks=self.lifecycle_hooks,
            init_containers=self._expand_all(self.init_containers),
            service_account=self.service_account,
            extra_container_config=self.extra_container_config,
            extra_pod_config=self.extra_pod_config,
            extra_containers=self._expand_all(self.extra_containers),
            scheduler_name=self.scheduler_name,
            tolerations=self.tolerations,
            node_affinity_preferred=self.node_affinity_preferred,
            node_affinity_required=self.node_affinity_required,
            pod_affinity_preferred=self.pod_affinity_preferred,
            pod_affinity_required=self.pod_affinity_required,
            pod_anti_affinity_preferred=self.pod_anti_affinity_preferred,
            pod_anti_affinity_required=self.pod_anti_affinity_required,
            priority_class_name=self.priority_class_name,
            logger=self.log,
        )
101228
def create(model_config, batch_size, normalize=True, num_workers=0, augmentations=None):
    
    path = model_config.data_dir('mnist')

    train_dataset = datasets.MNIST(path, train=True, download=True)
    test_dataset = datasets.MNIST(path, train=False, download=True)

    augmentations = [ToArray()] + (augmentations if augmentations is not None else [])

    if normalize:
        train_data = train_dataset.train_data
        mean_value = (train_data.double() / 255).mean().item()
        std_value = (train_data.double() / 255).std().item()

        augmentations.append(Normalize(mean=mean_value, std=std_value, tags=['train', 'val']))

    augmentations.append(ToTensor())

    return TrainingData(
        train_dataset,
        test_dataset,
        num_workers=num_workers,
        batch_size=batch_size,
        augmentations=augmentations
    )
6184
def build(capabilities, device_handler):
        ""
        if device_handler:
            # This is used as kwargs dictionary for lxml's Element() function.
            # Therefore the arg-name ("nsmap") is used as key here.
            xml_namespace_kwargs = { "nsmap" : device_handler.get_xml_base_namespace_dict() }
        else:
            xml_namespace_kwargs = {}
        hello = new_ele("hello", **xml_namespace_kwargs)
        caps = sub_ele(hello, "capabilities")
        def fun(uri): sub_ele(caps, "capability").text = uri
        #python3 changes
        if sys.version < '3':
            map(fun, capabilities)
        else:
            list(map(fun, capabilities))
        return to_xml(hello)
1342652
def build_model():
  
  # training data
  train_data_node = tf.placeholder(tf.float32,shape=(BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS))
  train_labels_node = tf.placeholder(tf.float32,shape=(BATCH_SIZE, NUM_LABELS))

  validation_data_node= tf.placeholder(tf.float32,shape=(VALIDATION_SIZE, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS))
  test_data_node=tf.placeholder(tf.float32,shape=(TEST_SIZE,IMAGE_SIZE,IMAGE_SIZE,NUM_CHANNELS))
  # validation dataset held in a single constant node
  # validation_data_node = tf.constant(validation_data)
  # test_data_node = tf.constant(test_data)

  # LEARNABLE WEIGHT NODES SHARED BETWEEN
  conv1_weights = tf.Variable(tf.truncated_normal([5, 5, NUM_CHANNELS, 32],stddev=0.1,seed=SEED))
  conv1_biases = tf.Variable(tf.zeros([32]))
  conv2_weights = tf.Variable(tf.truncated_normal([5, 5, 32, 64],stddev=0.1,seed=SEED))
  conv2_biases = tf.Variable(tf.constant(0.1, shape=[64]))
  fc1_weights = tf.Variable(tf.truncated_normal([IMAGE_SIZE // 4 * IMAGE_SIZE // 4 * 64, 512],stddev=0.1,seed=SEED))
  fc1_biases = tf.Variable(tf.constant(0.1, shape=[512]))
  fc2_weights = tf.Variable(tf.truncated_normal([512, NUM_LABELS],stddev=0.1,seed=SEED))
  fc2_biases = tf.Variable(tf.constant(0.1, shape=[NUM_LABELS]))

  # LENET
  def build_lenet(data,train=False):
    # subroutine for wiring up nodes and weights to training and evaluation LeNets
    conv1 = tf.nn.conv2d(data,conv1_weights,strides=[1, 1, 1, 1],padding='SAME')
    relu1 = tf.nn.relu(tf.nn.bias_add(conv1, conv1_biases))
    pool1 = tf.nn.max_pool(relu1,ksize=[1, 2, 2, 1],strides=[1, 2, 2, 1],padding='SAME')
    conv2 = tf.nn.conv2d(pool1,conv2_weights,strides=[1, 1, 1, 1],padding='SAME')
    relu2 = tf.nn.relu(tf.nn.bias_add(conv2, conv2_biases))
    pool2 = tf.nn.max_pool(relu2,ksize=[1, 2, 2, 1],strides=[1, 2, 2, 1],padding='SAME')
    # Reshape the feature map cuboid into a 2D matrix to feed it to the
    # fully connected layers.
    pool_shape = pool2.get_shape().as_list()
    reshape = tf.reshape(pool2,[pool_shape[0], pool_shape[1] * pool_shape[2] * pool_shape[3]])
    fc1 = tf.nn.relu(tf.matmul(reshape, fc1_weights) + fc1_biases)
    # Add a 50% dropout during training only. Dropout also scales
    # activations such that no rescaling is needed at evaluation time.
    if train:
      fc1 = tf.nn.dropout(fc1, 0.5, seed=SEED)
      # append summary ops to train
      _activation_summary(conv1)
      _activation_summary(fc1)

    fc2 = tf.matmul(fc1, fc2_weights) + fc2_biases
    return fc2

  # TRAINING LOSS / REGULARIZATION NODES
  logits = build_lenet(train_data_node, True)
  loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, train_labels_node))

  tf.scalar_summary(loss.op.name,loss)

  regularizers = (tf.nn.l2_loss(fc1_weights) + tf.nn.l2_loss(fc1_biases) + tf.nn.l2_loss(fc2_weights) + tf.nn.l2_loss(fc2_biases))
  # Add the regularization term to the loss.
  loss += 5e-4 * regularizers

  # OPTIMIZER NODES
  batch = tf.Variable(0)
  # Decay once per epoch, using an exponential schedule starting at 0.01.
  learning_rate = tf.train.exponential_decay(
    0.01,                # Base learning rate.
    batch * BATCH_SIZE,  # Current index into the dataset.
    TRAIN_SIZE,          # Decay step.
    0.95,                # Decay rate.
    staircase=True)
  # Use simple momentum for the optimization.
  optimizer = tf.train.MomentumOptimizer(learning_rate,0.9).minimize(loss,global_step=batch)

  # # Predictions for the minibatch, validation set and test set.
  train_prediction = tf.nn.softmax(logits)
  # # We'll compute them only once in a while by calling their {eval()} method.
  validation_prediction = tf.nn.softmax(build_lenet(validation_data_node))
  test_prediction = tf.nn.softmax(build_lenet(test_data_node))

  summaries=tf.merge_all_summaries()

  # return input nodes and output nodes
  return (train_data_node,
    train_labels_node,
    validation_data_node,
    test_data_node,
    train_prediction,
    validation_prediction,
    test_prediction,
    conv1_weights,
    conv2_weights,
    fc1_weights,
    fc2_weights,
    optimizer,
    loss,
    learning_rate,
    summaries)