Create runlist and upload it into the storage.
12290
def runlist_create(name, **kwargs):
    
    ctx = Context(**kwargs)
    ctx.execute_action('runlist:create', **{
        'storage': ctx.repo.create_secure_service('storage'),
        'name': name,
    })
1272027
def store_list(self, cb=None):
        """"""

        from StringIO import StringIO
        import json

        d = {}

        for k, v in self.list().items():

            if 'caches' in v:
                del v['caches']

            d[k] = v

        strio = StringIO(json.dumps(d))

        sink = self.put_stream('meta/_list.json')

        copy_file_or_flo(strio, sink, cb=cb)
        sink.close()
448184
def create(self):
        """"""
        self._storage_broker.create_structure()
        self._storage_broker.put_admin_metadata(self._admin_metadata)
727690
def runlist_view(name, **kwargs):
    
    ctx = Context(**kwargs)
    ctx.execute_action('runlist:view', **{
        'storage': ctx.repo.create_secure_service('storage'),
        'name': name
    })
675642
def build_from_job_list(scheme_files, templates, base_output_dir):
    """"""
    queue = Queue()
    for scheme in scheme_files:
        queue.put(scheme)

    if len(scheme_files) < 40:
        thread_num = len(scheme_files)
    else:
        thread_num = 40

    threads = []
    for _ in range(thread_num):
        thread = Thread(target=build_single_worker,
                        args=(queue, templates, base_output_dir))
        thread.start()
        threads.append(thread)

    queue.join()

    for _ in range(thread_num):
        queue.put(None)

    for thread in threads:
        thread.join()
734111
def setup_run(self):
        
        super(ContainerState, self).setup_run()
        # reset the scoped data
        self._scoped_data = {}
        self._start_state_modified = False
        self.add_default_values_of_scoped_variables_to_scoped_data()
        self.add_input_data_to_scoped_data(self.input_data)
701393
def prepare_runtime(self, runtime_dir, data):
        """"""
        # Copy over Python process runtime (resolwe.process).
        import resolwe.process as runtime_package

        src_dir = os.path.dirname(inspect.getsourcefile(runtime_package))
        dest_package_dir = os.path.join(runtime_dir, PYTHON_RUNTIME_DIRNAME, 'resolwe', 'process')
        shutil.copytree(src_dir, dest_package_dir)
        os.chmod(dest_package_dir, 0o755)

        # Write python source file.
        source = data.process.run.get('program', '')
        program_path = os.path.join(runtime_dir, PYTHON_PROGRAM_FILENAME)
        with open(program_path, 'w') as file:
            file.write(source)
        os.chmod(program_path, 0o755)

        # Write serialized inputs.
        inputs = copy.deepcopy(data.input)
        hydrate_input_references(inputs, data.process.input_schema)
        hydrate_input_uploads(inputs, data.process.input_schema)
        inputs_path = os.path.join(runtime_dir, PYTHON_INPUTS_FILENAME)

        # XXX: Skip serialization of LazyStorageJSON. We should support
        # LazyStorageJSON in Python processes on the new communication protocol
        def default(obj):
            """Get default value."""
            class_name = obj.__class__.__name__
            if class_name == 'LazyStorageJSON':
                return ''

            raise TypeError(f'Object of type {class_name} is not JSON serializable')

        with open(inputs_path, 'w') as file:
            json.dump(inputs, file, default=default)

        # Generate volume maps required to expose needed files.
        volume_maps = {
            PYTHON_RUNTIME_DIRNAME: PYTHON_RUNTIME_VOLUME,
            PYTHON_PROGRAM_FILENAME: PYTHON_PROGRAM_VOLUME,
            PYTHON_INPUTS_FILENAME: PYTHON_INPUTS_VOLUME,
        }

        return volume_maps
1281220
def run():
    """"""
    
    # Grab server args
    args = client_helper.grab_server_args()

    # Start up workbench connection
    workbench = zerorpc.Client(timeout=300, heartbeat=60)
    workbench.connect('tcp://'+args['server']+':'+args['port'])

    # Upload the files into workbench
    my_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                           '../data/pcap/gold_xxx.pcap')
    with open(my_file,'rb') as f:

        # We're going to upload the file in chunks to workbench 
        filename = os.path.basename(my_file)
        raw_bytes = f.read()
        md5_list = []
        for chunk in chunks(raw_bytes, 1024*1024):
            md5_list.append(workbench.store_sample(chunk, filename, 'exe'))

        # Now we just ask Workbench to combine these
        combined_md5 = workbench.combine_samples(md5_list, filename, 'exe')
        real_md5 = workbench.store_sample(raw_bytes, filename, 'exe')
        assert(combined_md5 == real_md5)
7211
def upload(self, *args, **kwargs):
    """"""
    for job in self.jobs:
      job.upload(*args, **kwargs)
463831
def setup():
    

    init_tasks()

    run_hook("before_setup")

    # Create shared folder
    env.run("mkdir -p %s" % (paths.get_shared_path()))
    env.run("chmod 755 %s" % (paths.get_shared_path()))

    # Create backup folder
    env.run("mkdir -p %s" % (paths.get_backup_path()))
    env.run("chmod 750 %s" % (paths.get_backup_path()))

    # Create uploads folder
    env.run("mkdir -p %s" % (paths.get_upload_path()))
    env.run("chmod 775 %s" % (paths.get_upload_path()))

    run_hook("setup")
    run_hook("after_setup")
450035
function List(runner) {
  Base.call(this, runner);

  var self = this
    , stats = this.stats
    , total = runner.total;

  runner.on('start', function(){
    console.log(JSON.stringify(['start', { total: total }]));
  });

  runner.on('pass', function(test){
    console.log(JSON.stringify(['pass', clean(test)]));
  });

  runner.on('fail', function(test, err){
    console.log(JSON.stringify(['fail', clean(test)]));
  });

  runner.on('end', function(){
    process.stdout.write(JSON.stringify(['end', self.stats]));
  });
}
1107511
def addList(self, source_id, dir_path, is_recurcive=False, timestamp_reception=None, training_metadata=[]):
        """"""
        if not path.isdir(dir_path):
            raise ValueError(dir_path + ' is not a directory')
        files_to_send = _get_files_from_dir(dir_path, is_recurcive)
        succeed_upload = {}
        failed_upload = {}
        for file_path in files_to_send:
            try:
                resp = self.add(source_id=source_id,
                    file_path=file_path, profile_reference="",
                    timestamp_reception=timestamp_reception, training_metadata=training_metadata)
                if resp['code'] != 200 and resp['code'] != 201:
                    failed_upload[file_path] = ValueError('Invalid response: ' + str(resp))
                else:
                    succeed_upload[file_path] = resp
            except BaseException as e:
                failed_upload[file_path] = e
        result = {
            'success': succeed_upload,
            'fail': failed_upload
        }
        return result
1312765
private void createImpl()
  {
    _systemSegment = allocateSegment(new byte[32], 1);
    
    _segmentList.add(_systemSegment);
    
    TempBuffer tBuf = TempBuffer.createLarge();
    byte []buffer = tBuf.buffer();
    
    Arrays.fill(buffer, (byte) 0);
    
    int offset = 0;
    
    BitsUtil.writeLong(buffer, offset, JOURNAL_MAGIC);
    offset += 8;
    
    BitsUtil.writeLong(buffer, offset, _segmentSize);
    offset += 8;
    
    BitsUtil.writeLong(buffer, offset, _tailAddress);
    offset += 8;
    
    BitsUtil.writeInt(buffer, offset, KEY_LENGTH);
    offset += 4;
    
    try (OutStore os = _store.openWrite(0, buffer.length)) {
      os.write(0, buffer, 0, buffer.length);
    }
    
    tBuf.free();
  }
286806
function runner (params, files) {
  var list = array(files || globs())

  list.forEach(function (file) {
    apply(require(path.resolve(file)), [test].concat(params))
  })

  return list
}
664116
def new_run(self):
        """"""
        self.current_run += 1
        self.runs.append(RunData(self.current_run + 1))
123692
def run(self, path, pretend=False):
        
        self._notes = []

        files = self._get_migration_files(path)

        ran = self._repository.get_ran()

        migrations = [f for f in files if f not in ran]

        self.run_migration_list(path, migrations, pretend)
589687
def _handle_runs(self, relpath, params):
    """"""
    runs_by_day = self._partition_runs_by_day()
    args = self._default_template_args('run_list.html')
    args['runs_by_day'] = runs_by_day
    content = self._renderer.render_name('base.html', args).encode("utf-8")
    self._send_content(content, 'text/html')