Simple push/pull streamer

    :param int frontend: fontend zeromq port
    :param int backend: backend zeromq port
354280
def streamer(frontend, backend):
    
    try:
        context = zmq.Context()

        front_pull = context.socket(zmq.PULL)
        front_pull.set_hwm(0)
        front_pull.bind("tcp://*:%d" % frontend)

        back_push = context.socket(zmq.PUSH)
        back_push.bind("tcp://*:%d" % backend)

        print("streamer started, backend on port : %d\tfrontend on port: %d" % (backend, frontend))
        zmq.proxy(front_pull, back_push)
    except Exception as e:
        print(e)
    finally:
        front_pull.close()
        back_push.close()
        context.term()
1254525
def pull(self):
        
        sock = self.__sock(zmq.PULL)
        return self.__recv_generator(sock)
691159
async def run(self, **params):
        
        if self.json_stream:
            warnings.warn("already running", RuntimeWarning, stackelevel=2)
            return
        forced_params = {"stream": True}
        params = ChainMap(forced_params, params)
        try:
            # timeout has to be set to 0, None is not passed
            # Otherwise after 5 minutes the client
            # will close the connection
            # http://aiohttp.readthedocs.io/en/stable/client_reference.html#aiohttp.ClientSession.request
            response = await self.docker._query(
                "events", method="GET", params=params, timeout=0
            )
            self.json_stream = await json_stream_result(
                response, self._transform_event, human_bool(params["stream"])
            )
            try:
                async for data in self.json_stream:
                    await self.channel.publish(data)
            finally:
                if self.json_stream is not None:
                    await self.json_stream._close()
                self.json_stream = None
        finally:
            # signal termination to subscribers
            await self.channel.publish(None)
794033
function ({ feedId, live = false, reverse = false }, cb) {
      var stream = PullPushAbort()

      var result = {}
      var sync = null

      // stream reverse states if option specified
      var queryStream = reverse ? ssb.backlinks.read({
        query: [{ $filter: {
          dest: feedId,
          value: {
            content: {
              type: 'contact',
              contact: feedId
            }
          }
        } }],
        live
      }) : ssb.query.read({
        query: [{ $filter: {
          value: {
            author: feedId,
            content: {
              type: 'contact'
            }
          }
        } }],
        live
      })

      pull(
        queryStream,
        stream.aborter,
        pull.filter(msg => (msg.value && msg.value.content && msg.value.content.type) || msg.sync),
        pull.drain((msg) => {
          if (msg.sync) {
            // send first reduced result when running in live mode
            sync = true
            if (live) stream.push(result)
            return
          }

          var isPrivate = msg.value && msg.value.meta && msg.value.meta.private && msg.value.author === ssb.id

          if (!isPrivate) {
            var content = msg.value.content
            var contact = reverse ? msg.value.author : content.contact
            var value = getContactState(msg.value.content)

            if (sync) {
              // send updated state in live mode
              stream.push({ [contact]: value })
            } else {
              result[contact] = value
            }
          }
        }, (err) => {
          if (err) return stream.end(err)
          if (stream.ended || sync) return

          // send final result when not live
          if (!live) stream.push(result)
          stream.end()
        })
      )

      return stream
    }
955094
def downstream(self, url, format):
        
        headers={'Authorization': 'Bearer '+self.token, 'x-falkonry-source':self.sourceHeader}
        if format is not None:
            headers['accept'] = format
        response = requests.get(self.host + url, stream=True, headers=headers, verify=False)
        if response.status_code == 200 or response.status_code == 202:
            return response
        elif response.status_code == 401:
            raise Exception(json.dumps({'message':'Unauthorized Access'}))
        else:
            raise Exception(json.dumps({'message':str(response.text)}))
811215
def show_backends(socket=DEFAULT_SOCKET_URL):
    '''
    
    '''
    ha_conn = _get_conn(socket)
    ha_cmd = haproxy.cmds.showBackends()
    return ha_conn.sendCmd(ha_cmd)
659226
def feed_backend(url, clean, fetch_archive, backend_name, backend_params,
                 es_index=None, es_index_enrich=None, project=None, arthur=False,
                 es_aliases=None, projects_json_repo=None):
    

    backend = None
    repo = {'backend_name': backend_name, 'backend_params': backend_params}  # repository data to be stored in conf

    if es_index:
        clean = False  # don't remove index, it could be shared

    if not get_connector_from_name(backend_name):
        raise RuntimeError("Unknown backend %s" % backend_name)
    connector = get_connector_from_name(backend_name)
    klass = connector[3]  # BackendCmd for the connector

    try:
        logger.info("Feeding Ocean from %s (%s)", backend_name, es_index)

        if not es_index:
            logger.error("Raw index not defined for %s", backend_name)

        repo['repo_update_start'] = datetime.now().isoformat()

        # perceval backends fetch params
        offset = None
        from_date = None
        category = None
        latest_items = None
        filter_classified = None

        backend_cmd = klass(*backend_params)

        parsed_args = vars(backend_cmd.parsed_args)
        init_args = find_signature_parameters(backend_cmd.BACKEND,
                                              parsed_args)

        if backend_cmd.archive_manager and fetch_archive:
            archive = Archive(parsed_args['archive_path'])
        else:
            archive = backend_cmd.archive_manager.create_archive() if backend_cmd.archive_manager else None

        init_args['archive'] = archive
        backend_cmd.backend = backend_cmd.BACKEND(**init_args)
        backend = backend_cmd.backend

        ocean_backend = connector[1](backend, fetch_archive=fetch_archive, project=project)
        elastic_ocean = get_elastic(url, es_index, clean, ocean_backend, es_aliases)
        ocean_backend.set_elastic(elastic_ocean)
        ocean_backend.set_projects_json_repo(projects_json_repo)

        if fetch_archive:
            signature = inspect.signature(backend.fetch_from_archive)
        else:
            signature = inspect.signature(backend.fetch)

        if 'from_date' in signature.parameters:
            try:
                # Support perceval pre and post BackendCommand refactoring
                from_date = backend_cmd.from_date
            except AttributeError:
                from_date = backend_cmd.parsed_args.from_date

        if 'offset' in signature.parameters:
            try:
                offset = backend_cmd.offset
            except AttributeError:
                offset = backend_cmd.parsed_args.offset

        if 'category' in signature.parameters:
            try:
                category = backend_cmd.category
            except AttributeError:
                try:
                    category = backend_cmd.parsed_args.category
                except AttributeError:
                    pass

        if 'filter_classified' in signature.parameters:
            try:
                filter_classified = backend_cmd.parsed_args.filter_classified
            except AttributeError:
                pass

        if 'latest_items' in signature.parameters:
            try:
                latest_items = backend_cmd.latest_items
            except AttributeError:
                latest_items = backend_cmd.parsed_args.latest_items

        # fetch params support
        if arthur:
            # If using arthur just provide the items generator to be used
            # to collect the items and upload to Elasticsearch
            aitems = feed_backend_arthur(backend_name, backend_params)
            ocean_backend.feed(arthur_items=aitems)
        else:
            params = {}
            if latest_items:
                params['latest_items'] = latest_items
            if category:
                params['category'] = category
            if filter_classified:
                params['filter_classified'] = filter_classified
            if from_date and (from_date.replace(tzinfo=None) != parser.parse("1970-01-01")):
                params['from_date'] = from_date
            if offset:
                params['from_offset'] = offset

            ocean_backend.feed(**params)

    except Exception as ex:
        if backend:
            logger.error("Error feeding ocean from %s (%s): %s", backend_name, backend.origin, ex, exc_info=True)
        else:
            logger.error("Error feeding ocean %s", ex, exc_info=True)

    logger.info("Done %s ", backend_name)
276415
def _create_listening_stream(self, pull_addr):
        
        sock = self._zmq_context.socket(zmq.PULL)
        sock.connect(pull_addr)
        stream = ZMQStream(sock, io_loop=self.io_loop)

        return stream
793511
function buildStream (host, options, secure) {
  var stream = [];

  // accept options as a secure boolean
  if (options === true) {
    secure = true;
  }

  options = _.defaults(_.isObject(options) ? options : {}, {
    request  : {},
    response : {}
  });

  function applyStreamFunctions (funcSet) {
    _.each(funcSet || [], function (middleware) {
      if (_.isFunction(middleware)) {
        // wrapper allows us to call the callback without passing proxyObj
        stream.push(function asyncWrapper (proxyObj, callback) {
          return middleware(proxyObj, (error, _proxyObj) => callback(error, _proxyObj || proxyObj));
        });
      }
    });
  }

  // if there is `pre` middleware
  let preSet = options.pre;
  if (preSet) {
    // accept raw functions, but wrap it in an array
    preSet = _.isFunction(preSet) ? [preSet] : preSet;
    applyStreamFunctions(preSet);
  }

  // push the core request
  stream.push(proxyReq);

  // if there is `post` middleware
  let postSet = options.post;
  if (postSet) {
    // accept raw functions, but wrap it in an array
    postSet = _.isFunction(postSet) ? [postSet] : postSet;
    applyStreamFunctions(postSet);
  }

  if (secure) {
    options.request.forceHttps = true;
  }

  return stream;
}
173753
def start_receive(self, stream):
        
        _vv and IOLOG.debug('%r.start_receive(%r)', self, stream)
        side = stream.receive_side
        assert side and side.fd is not None
        self.defer(self.poller.start_receive,
                   side.fd, (side, stream.on_receive))
486659
function createStreamListenersWrap(api) {
  return function streamListenersWrap(install_stream_listeners) {
    return function install_stream_listeners_trace() {
      api.wrapEmitter(this.stream);
      return install_stream_listeners.apply(this, arguments);
    };
  };
}
765877
function Stream (options) {

  //check options
  if (!options) {

    throw new Error('Initialization error: missing transport options')
  }

  this._initDefaults() //Initialize default params
  this._updateParams(options)
  this._initTransport(options.transport)

  var self = this

  self.on('error', nop)

  this.subscriptions = []

  this.getSubscriptions = function () {

    return this.subscriptions
  }

  this.addSubscription = function (args, callback) {

    var c = new Subscription(args, callback, this)
    this.subscriptions[c.eid] = c
    return c
  }

  this.getSubscription = function (channel, resource) {

    if (this.subscriptions[channel + this.seperator + resource]) {
      return this.subscriptions[channel + this.seperator + resource]
    }
    return null
  }

  this.getAnySubscription = function (channel, resource) {

    if (this.subscriptions[channel + this.seperator + resource]) {
      return this.subscriptions[channel + this.seperator + resource]
    } else if (this.subscriptions[channel + this.seperator + this.wildcard]) {
      return this.subscriptions[channel + this.seperator + this.wildcard]
    } else {
      return null
    }
  }

  this.removeSubscription = function (channel, resource) {

    resource = resource || this.wildcard
    if (this.subscriptions[channel + this.seperator + resource]) {
      var retval = this.subscriptions[channel + this.seperator + resource]
      delete this.subscriptions[channel + this.seperator + resource]
      return retval
    }

    return null
  }

  this.getSubscriptionWithPermission = function (channel, resource, read, write) {

    var Subscription = this.subscriptions[channel + this.seperator + resource]
    var match = false

    if (Subscription) {

      match = true

      if (read) {
        match = Subscription.hasReadPermission()
      }

      if (write) {
        match = Subscription.hasWritePermission()
      }

      if (match) {
        return Subscription
      }
    } else {

      Subscription = this.subscriptions[channel + this.seperator + this.wildcard]

      if (Subscription) {

        match = true

        if (read) {
          match = Subscription.hasReadPermission()
        }

        if (write) {
          match = Subscription.hasWritePermission()
        }

        if (match) {
          return Subscription
        }
      }
    }

    return null
  }

  this.connection = new this.ConnectionFactory(this)
  this.connect()
}
1345914
def stream_upload(self, data, callback):
        
        while True:
            chunk = data.read(self.config.connection.data_block_size)
            if not chunk:
                break
            if callback and callable(callback):
                callback(chunk, response=None)
            yield chunk
349570
function streamer(error, output, callback)
{
  if (error && !this.error)
  {
    this.error = error;
    this.pause();
    this.emit('error', error);
    // send back value only, as expected
    callback(error, output && output.value);
    return;
  }

  // stream stuff
  this.push(output);

  // back to original track
  // send back value only, as expected
  callback(error, output && output.value);
}
607115
def _work(self, backend, package, ident='', log=True):
        
        num = self._sending_task(backend)
        if log:
            self.log(INFO, 'Starting %s backend task #%s (%s)' % 
                           (backend, num, ident))
        try:
            conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            conn.connect(backend)
            self.send(conn, package)
            result = self.recv(conn)
            self.log(DEBUG, result)
            result = self.codec.decode(result)
        except Exception as e:
            self._canceling_task(backend)
            raise
        else:
            self._closing_task(backend)
            if log:
                self.log(INFO, 'Finished %s backend task #%s (%s)' % 
                               (backend, num, ident))
        finally:
            conn.close()
        
        if result[0] == 'error':
            # We reraise errors in our own way.
            raise BackendProcessingError(*result[1:])
        else:
            return result[1]
863783
def create_app(cls, port):
        
        app = FrontEndApp()
        app_server = GeventServer(app, port=port, hostname='0.0.0.0')
        return app_server
1034017
def stream(self, flags=0, devpath=None):
        ""
        if flags & WATCH_DISABLE:
            arg = '?WATCH={"enable":false'
            if flags & WATCH_JSON:
                arg += ',"json":false'
            if flags & WATCH_NMEA:
                arg += ',"nmea":false'
            if flags & WATCH_RARE:
                arg += ',"raw":1'
            if flags & WATCH_RAW:
                arg += ',"raw":2'
            if flags & WATCH_SCALED:
                arg += ',"scaled":false'
            if flags & WATCH_TIMING:
                arg += ',"timing":false'
        else: # flags & WATCH_ENABLE:
            arg = '?WATCH={"enable":true'
            if flags & WATCH_JSON:
                arg += ',"json":true'
            if flags & WATCH_NMEA:
                arg += ',"nmea":true'
            if flags & WATCH_RAW:
                arg += ',"raw":1'
            if flags & WATCH_RARE:
                arg += ',"raw":0'
            if flags & WATCH_SCALED:
                arg += ',"scaled":true'
            if flags & WATCH_TIMING:
                arg += ',"timing":true'
            if flags & WATCH_DEVICE:
                arg += ',"device":"%s"' % devpath
        return self.send(arg + "}")