creates data point payloads for trending data to influxdb
4569
def send_trending_data(events):
    
    bodies = {}

    # sort the values
    top_hits = sorted(
        [(key, count) for key, count in events.items()],
        key=lambda x: x[1],
        reverse=True
    )[:100]

    # build up points to be written
    for (site, content_id), count in top_hits:
        if not len(site) or not re.match(CONTENT_ID_REGEX, content_id):
            continue

        # add point
        bodies.setdefault(site, [])
        bodies[site].append([content_id, count])

    for site, points in bodies.items():
        # create name
        name = "{}_trending".format(site)
        # send payload to influxdb
        try:
            data = [{
                "name": name,
                "columns": ["content_id", "value"],
                "points": points,
            }]
            INFLUXDB_CLIENT.write_points(data)
        except Exception as e:
            LOGGER.exception(e)
484088
def trending(params):
    
    # get params
    try:
        series = params.get("site", [DEFAULT_SERIES])[0]
        offset = params.get("offset", [DEFAULT_GROUP_BY])[0]
        limit = params.get("limit", [20])[0]
    except Exception as e:
        LOGGER.exception(e)
        return json.dumps({"error": e.message}), "500 Internal Error"

    # check the cache
    cache_key = "{}:{}:{}:{}:{}".format(memcached_prefix, "trending.json", series, offset, limit)
    try:
        data = MEMCACHED_CLIENT.get(cache_key)
        if data:
            return data, "200 OK"
    except Exception as e:
        LOGGER.exception(e)

    # update series name
    series = update_trending_series(series)

    # parse the limit
    try:
        limit = int(limit)
    except ValueError:
        LOGGER.error("limit param must be an integer")
        return json.dumps({"error": "limit param must be an integer"}), "400 Bad Request"

    # build the query
    query = "SELECT content_id, sum(value) as value " \
            "FROM {series} " \
            "WHERE time > now() - {offset} " \
            "GROUP BY content_id;"
    args = {"series": series, "offset": offset}

    # send the request
    try:
        res = INFLUXDB_CLIENT.query(query.format(**args))

    # capture errors and send them back along with the query (for inspection/debugging)
    except Exception as e:
        LOGGER.exception(e)
        return json.dumps({"error": e.message, "query": query.format(**args)}), "500 Internal Error"

    # build the response object
    response = flatten_response(res)

    # limit the number of content per site
    for site, points in response.items():
        sorted_content = sorted(points, key=lambda p: p["value"], reverse=True)[:limit]
        response[site] = sorted_content

    clean_response = {}
    for site, values in response.items():
        clean_name = site.split("-")[0]
        clean_response[clean_name] = values
    res = json.dumps(clean_response)

    # cache the response
    try:
        MEMCACHED_CLIENT.set(cache_key, res, time=MEMCACHED_EXPIRATION)
    except Exception as e:
        LOGGER.exception(e)

    return res, "200 OK"
73795
def main(uport):
    """"""
    # NOTE: structure of the UDP packet is different than that of information
    #       sent via HTTP
    json_body = {
        "tags": {
            "host": "server01",
            "region": "us-west"
        },
        "time": "2009-11-10T23:00:00Z",
        "points": [{
            "measurement": "cpu_load_short",
            "fields": {
                "value": 0.64
            }
        },
            {
                "measurement": "cpu_load_short",
                "fields": {
                    "value": 0.67
                }
        }]
    }

    # make `use_udp` True and  add `udp_port` number from `influxdb.conf` file
    # no need to mention the database name since it is already configured
    client = InfluxDBClient(use_udp=True, udp_port=uport)

    # Instead of `write_points` use `send_packet`
    client.send_packet(json_body)
1324543
def metric_details(request, pk, format=None):
    
    metric = get_object_or_404(Metric, pk=pk)
    # get
    if request.method == 'GET':
        try:
            results = metric.select(q=request.query_params.get('q', metric.query))
        except InfluxDBClientError as e:
            return Response({'detail': e.content}, status=e.code)
        return Response(list(results.get_points(metric.name)))
    # post
    else:
        if not request.data:
            return Response({'detail': 'expected values in POST data or JSON payload'},
                            status=400)
        data = request.data.copy()
        # try converting strings to floats when sending form-data
        if request.content_type != 'application/json':
            for key, value in data.items():
                try:
                    data[key] = float(value) if '.' in value else int(value)
                except ValueError:
                    pass
        # write
        metric.write(data)
        return Response({'detail': 'ok'})
1180091
def write_points_with_precision(self, data, time_precision='s'):
        
        warnings.warn(
            "write_points_with_precision is deprecated, and will be removed "
            "in future versions. Please use "
            "``InfluxDBClient.write_points(time_precision='..')`` instead.",
            FutureWarning)
        return self._write_points(data=data, time_precision=time_precision)
1327238
def _make_graphite_api_points_list(influxdb_data):
    """"""
    _data = {}
    for key in influxdb_data.keys():
        _data[key[0]] = [(datetime.datetime.fromtimestamp(float(d['time'])),
                          d['value']) for d in influxdb_data.get_points(key[0])]
    return _data
835868
def export(self, name, columns, points):
        """"""
        data = ('hostname=' + self.hostname + ', name=' + name +
                ', dateinfo=' + datetime.datetime.utcnow().isoformat())
        for i in range(len(columns)):
            if not isinstance(points[i], Number):
                continue
            else:
                data += ", " + columns[i] + "=" + str(points[i])
        logger.debug(data)
        try:
            self.client.basic_publish(exchange='', routing_key=self.queue, body=data)
        except Exception as e:
            logger.error("Can not export stats to RabbitMQ (%s)" % e)
150676
function makePoint() {
      // Pull start time from req and log responseTime
      var responseTime = Date.now() - req.start;
      
      // Add the new point to the batch of points
      batch.points.push({
        measurement: "requests",
        "tags": {
          "path": req.path,
          "host": req.hostname,
          "verb": req.method,
          "status": res.statusCode,
        },
        "fields": {
          "responseTime": responseTime,
        },
      })
      
      // Emit the 'addPoint' event
      batch.emit("addPoint")
    }
867234
def export(self, name, columns, points):
        """"""
        logger.debug("Export {} stats to Kafka".format(name))

        # Create DB input
        data = dict(zip(columns, points))

        # Send stats to the kafka topic
        # key=<plugin name>
        # value=JSON dict
        try:
            self.client.send(self.topic,
                             key=name,
                             value=data)
        except Exception as e:
            logger.error("Cannot export {} stats to Kafka ({})".format(name, e))
955783
def export(self, name, columns, points):
        """"""
        # Manage prefix
        if self.prefix is not None:
            name = self.prefix + '.' + name
        # Write input to the InfluxDB database
        try:
            self.client.write_points(self._normalize(name, columns, points))
        except Exception as e:
            logger.error("Cannot export {} stats to InfluxDB ({})".format(name,
                                                                          e))
        else:
            logger.debug("Export {} stats to InfluxDB".format(name))
490112
def main(host='localhost', port=8086, nb_day=15):
    """"""
    nb_day = 15  # number of day to generate time series
    timeinterval_min = 5  # create an event every x minutes
    total_minutes = 1440 * nb_day
    total_records = int(total_minutes / timeinterval_min)
    now = datetime.datetime.today()
    metric = "server_data.cpu_idle"
    series = []

    for i in range(0, total_records):
        past_date = now - datetime.timedelta(minutes=i * timeinterval_min)
        value = random.randint(0, 200)
        hostName = "server-%d" % random.randint(1, 5)
        # pointValues = [int(past_date.strftime('%s')), value, hostName]
        pointValues = {
            "time": int(past_date.strftime('%s')),
            "measurement": metric,
            "fields": {
                "value": value,
            },
            "tags": {
                "hostName": hostName,
            },
        }
        series.append(pointValues)

    print(series)

    client = InfluxDBClient(host, port, USER, PASSWORD, DBNAME)

    print("Create database: " + DBNAME)
    try:
        client.create_database(DBNAME)
    except InfluxDBClientError:
        # Drop and create
        client.drop_database(DBNAME)
        client.create_database(DBNAME)

    print("Create a retention policy")
    retention_policy = 'server_data'
    client.create_retention_policy(retention_policy, '3d', 3, default=True)

    print("Write points #: {0}".format(total_records))
    client.write_points(series, retention_policy=retention_policy)

    time.sleep(2)

    query = "SELECT MEAN(value) FROM {} WHERE \
            time > now() - 10d GROUP BY time(500m)".format(metric)
    result = client.query(query, database=DBNAME)
    print(result)
    print("Result: {0}".format(result))

    print("Drop database: {}".format(DBNAME))
    client.drop_database(DBNAME)
260005
def get_output_values(self):
        
        # Handle primitive values here and implicitly convert them to a dict because
        # it allows the API to be simpler.

        # Also influxDB mandates that each value also has a name so the default name
        # for any non-dict value is "value"
        if not isinstance(self.values, dict):
            metric_values = {'value': self.values}
        else:
            metric_values = self.values

        # Sort the values in lexicographically by value name
        sorted_values = sorted(metric_values.items())

        # Remove None values
        sorted_values = [(k, v) for k, v in sorted_values if v is not None]

        return u",".join(u"{0}={1}".format(format_string(k), format_value(v)) for k, v in sorted_values)
1119387
def query(query, params={}, epoch=None,
          expected_response_code=200, database=None):
    """"""
    db = get_db()
    database = database or settings.INFLUXDB_DATABASE
    return db.query(query, params, epoch, expected_response_code, database=database)
233212
def createExperiment(uuid: str):
    
    data = request.get_json()
    G = DelphiModel.query.filter_by(id=uuid).first().model
    if os.environ.get("TRAVIS") is not None:
        config_file="bmi_config.txt"
    else:
        if not os.path.exists("/tmp/delphi"):
            os.makedirs("/tmp/delphi", exist_ok=True)
        config_file="/tmp/delphi/bmi_config.txt"

    G.initialize(initialize_indicators = False, config_file=config_file)
    for n in G.nodes(data=True):
        rv = n[1]["rv"]
        rv.partial_t = 0.0
        for variable in data["interventions"]:
            if n[1]["id"] == variable["id"]:
                # TODO : Right now, we are only taking the first value in the
                # "values" list. Need to generalize this so that you can have
                # multiple interventions at different times.

                # TODO : The subtraction of 1 is a TEMPORARY PATCH to address
                # the mismatch in semantics between the ICM API and the Delphi
                # model. MUST FIX ASAP.
                rv.partial_t = variable["values"]["value"]["value"] - 1
                for s0 in G.s0:
                    s0[f"∂({n[0]})/∂t"] = rv.partial_t
                break

    id = str(uuid4())
    experiment = ForwardProjection(baseType="ForwardProjection", id=id)
    db.session.add(experiment)
    db.session.commit()

    result = ForwardProjectionResult(id=id, baseType="ForwardProjectionResult")
    db.session.add(result)
    db.session.commit()

    d = dateutil.parser.parse(data["projection"]["startTime"])

    n_timesteps = data["projection"]["numSteps"]

    for i in range(n_timesteps):
        if data["projection"]["stepSize"] == "MONTH":
            d = d + relativedelta(months=1)
        elif data["projection"]["stepSize"] == "YEAR":
            d = d + relativedelta(years=1)

        for n in G.nodes(data=True):
            CausalVariable.query.filter_by(
                id=n[1]["id"]
            ).first().lastUpdated = d.isoformat()
            result.results.append(
                {
                    "id": n[1]["id"],
                    "baseline": {
                        "active": "ACTIVE",
                        "time": d.isoformat(),
                        "value": {"baseType": "FloatValue", "value": 1.0},
                    },
                    "intervened": {
                        "active": "ACTIVE",
                        "time": d.isoformat(),
                        "value": {
                            "baseType": "FloatValue",
                            "value": np.median([s[n[0]] for s in G.s0]),
                        },
                    },
                }
            )

        G.update(update_indicators=False)

        # Hack for 12-month evaluation - have the partial derivative decay over
        # time to restore equilibrium

        tau = 1.0 # Time constant to control the rate of the decay
        for n in G.nodes(data=True):
            for variable in data["interventions"]:
                if n[1]["id"] == variable["id"]:
                    rv = n[1]["rv"]
                    for s0 in G.s0:
                        s0[f"∂({n[0]})/∂t"] = rv.partial_t * exp(-tau*i)

    db.session.add(result)
    db.session.commit()

    return jsonify(
        {
            "id": experiment.id,
            "message": "Forward projection sent successfully",
        }
    )
175900
def export(self, name, columns, points):
        """"""
        logger.debug("Export {} stats to ZeroMQ".format(name))

        # Create DB input
        data = dict(zip(columns, points))

        # Do not publish empty stats
        if data == {}:
            return False

        # Glances envelopes the stats in a publish message with two frames:
        # - First frame containing the following prefix (STRING)
        # - Second frame with the Glances plugin name (STRING)
        # - Third frame with the Glances plugin stats (JSON)
        message = [b(self.prefix),
                   b(name),
                   asbytes(json.dumps(data))]

        # Write data to the ZeroMQ bus
        # Result can be view: tcp://host:port
        try:
            self.client.send_multipart(message)
        except Exception as e:
            logger.error("Cannot export {} stats to ZeroMQ ({})".format(name, e))

        return True
611013
def into_signalfx_point(self, metric):
        
        dims = {
            "collector": metric.getCollectorPath(),
            "prefix": metric.getPathPrefix(),
        }
        if metric.host is not None and metric.host != "":
            dims["host"] = metric.host

        return {
            "metric": metric.getMetricPath(),
            "value": metric.value,
            "dimensions": dims,
            # We expect ms timestamps
            "timestamp": metric.timestamp * 1000,
        }
1033166
def to_sql(self, app=None, last_known_value_date: Optional[date] = None):
        

        from delphi.apps.rest_api import create_app, db

        self.assign_uuids_to_nodes_and_edges()
        icm_metadata = ICMMetadata(
            id=self.id,
            created=self.dateCreated,
            estimatedNumberOfPrimitives=len(self.nodes) + len(self.edges),
            createdByUser_id=1,
            lastAccessedByUser_id=1,
            lastUpdatedByUser_id=1,
        )
        if last_known_value_date is None:
            today = date.today().isoformat()
        else:
            today = last_known_value_date.isoformat()
        default_latent_var_value = 1.0
        causal_primitives = []
        nodeset = {n.split("/")[-1] for n in self.nodes}
        simplified_labels = len(nodeset) == len(self)
        for n in self.nodes(data=True):
            n[1]["rv"] = LatentVar(n[0])
            n[1]["update_function"] = self.default_update_function
            rv = n[1]["rv"]
            rv.dataset = [default_latent_var_value for _ in range(self.res)]

            causal_variable = CausalVariable(
                id=n[1]["id"],
                model_id=self.id,
                units="",
                namespaces={},
                auxiliaryProperties=[],
                label=n[0].split("/")[-1].replace("_", " ").capitalize()
                if simplified_labels
                else n[0],
                description=n[0],
                lastUpdated=today,
                confidence=1.0,
                lastKnownValue={
                    "active": "ACTIVE",
                    "trend": None,
                    "time": today,
                    "value": {
                        "baseType": "FloatValue",
                        "value": n[1]["rv"].dataset[0],
                    },
                },
                range={
                    "baseType": "FloatRange",
                    "range": {"min": 0, "max": 5, "step": 0.1},
                },
            )
            causal_primitives.append(causal_variable)

        max_mean_betas = max(
            [abs(np.median(e[2]["βs"])) for e in self.edges(data=True)]
        )
        for e in self.edges(data=True):
            causal_relationship = CausalRelationship(
                id=e[2]["id"],
                namespaces={},
                source={
                    "id": self.nodes[e[0]]["id"],
                    "baseType": "CausalVariable",
                },
                target={
                    "id": self.nodes[e[1]]["id"],
                    "baseType": "CausalVariable",
                },
                model_id=self.id,
                auxiliaryProperties=[],
                lastUpdated=today,
                types=["causal"],
                description="",
                confidence=np.mean(
                    [s.belief for s in e[2]["InfluenceStatements"]]
                ),
                label="",
                strength=abs(np.median(e[2]["βs"]) / max_mean_betas),
                reinforcement=(
                    True
                    if np.mean(
                        [
                            stmt.subj_delta["polarity"]
                            * stmt.obj_delta["polarity"]
                            for stmt in e[2]["InfluenceStatements"]
                        ]
                    )
                    > 0
                    else False
                ),
            )
            causal_primitives.append(causal_relationship)
        evidences = []
        for edge in self.edges(data=True):
            for stmt in edge[2]["InfluenceStatements"]:
                for ev in stmt.evidence:
                    evidence = Evidence(
                        id=str(uuid4()),
                        causalrelationship_id=edge[2]["id"],
                        # TODO - make source and target appear in CauseEx HMI
                        description=(ev.text),
                    )
                    evidences.append(evidence)

        if app is None:
            app = create_app()

        with app.app_context():
            db.drop_all()
            db.create_all()
            db.session.add(icm_metadata)
            db.session.add(DelphiModel(id=self.id, model=self))
            for causal_primitive in causal_primitives:
                db.session.add(causal_primitive)
            for evidence in evidences:
                db.session.add(evidence)
            db.session.commit()