# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # #02 - Exploring Preprocessing 

# Data preprocessing is a set of activities performed to prepare data for future analysis and data mining activities.

# ## Load data from file

# The dataset used in this tutorial is GeoLife GPS Trajectories. Available in https://www.microsoft.com/en-us/download/details.aspx?id=52367

import pandas as pd
import numpy as np
from pymove import MoveDataFrame

df = pd.read_csv('geolife_sample.csv', parse_dates=['datetime'])
df.head()

df_move = MoveDataFrame(df, latitude="lat", longitude="lon", datetime="datetime")

df_move.show_trajectories_info()

# ## Filtering

# The filters module provides functions to perform different types of data filtering.

# Importing the module:

from pymove import filters

# A bounding box (usually shortened to bbox) is an area defined by two longitudes and two latitudes. The function by_bbox, filters points of the trajectories according to a especified bounding box.

bbox = (22.147577, 113.54884299999999, 41.132062, 121.156224)
filt_df = filters.by_bbox(df_move, bbox)
filt_df.head()

# by_datetime function filters point trajectories according to the time specified by the parameters: start_datetime and end_datetime.

filters.by_datetime(df_move, start_datetime = "2009-03-19 05:45:37", end_datetime = "2009-03-19 05:46:17")

# by label function filters trajectories points according to specified value and column label, set by value and label_name respectively.

filters.by_label(df_move, value = 116.327219, label_name = "lon").head()

# by_id function filters trajectories points according to especified trajectory id.

filters.by_id(df_move, id_=5).head()

# A tid is the result of concatenation between the id and date of a trajectory.
# The by_tid function filters trajectory points according to the tid specified by the tid_ parameter.

df_move.generate_tid_based_on_id_datatime()
filters.by_tid(df_move, "12008102305").head()

# outliers function filters trajectories points that are outliers.

outliers_points = filters.outliers(df_move)
outliers_points.head()

# clen_duplicates function removes the duplicate rows of the Dataframe, optionally only certaind columns can be consider.

filters.clean_duplicates(df_move)

# clean_consecutive_duplicates function removes consecutives duplicate rows of the Dataframe. Optionally only certaind columns can be consider, this is defined by the parameter subset, in this example only the lat column is considered.

filtered_df = filters.clean_consecutive_duplicates(df_move, subset = ["lat"])
len(filtered_df)

# clean_nan_values function removes missing values from the dataframe.

filters.clean_nan_values(df_move)
len(df_move)

# clean_gps_jumps_by_distance function removes from the dataframe the trajectories points that are outliers.

filters.clean_gps_jumps_by_distance(df_move)

# clean_gps_nearby_points_by_distances function removes points from the trajectories when the distance between them and the point before is smaller than the parameter radius_area.

filters.clean_gps_nearby_points_by_distances(df_move, radius_area = 10)

# clean_gps_nearby_points_by_speed function removes points from the trajectories when the speed of travel between them
# and the point before is smaller than the value set by the parameter speed_radius.

filters.clean_gps_nearby_points_by_speed(df_move, speed_radius=40.0)

# clean_gps_speed_max_radius function recursively removes trajectories points with speed higher than the value especifeid by the user.
#     Given any point p of the trajectory, the point will be removed if one of the following happens:
#     if the travel speed from the point before p to p is greater than the  max value of speed between adjacent
#     points set by the user. Or the travel speed between point p and the next point is greater than the value set by
#     the user. When the clening is done, the function will update the time and distance features in the dataframe and
#     will call itself again.
#     The function will finish processing when it can no longer find points disrespecting the limit of speed.

filters.clean_gps_speed_max_radius(df_move)

# clean_trajectories_with_few_points function removes from the given dataframe, trajectories with fewer points than was specified by the parameter min_points_per_trajectory.

filters.clean_trajectories_with_few_points(df_move)

# ## Segmantation

# The segmentation module are used to segment trajectories based on different parameters.

# Importing the module:

from pymove import segmentation

# bbox_split function splits the bounding box in grids of the same size. The number of grids is defined by the parameter number_grids.

bbox = (22.147577, 113.54884299999999, 41.132062, 121.156224)
segmentation.bbox_split(bbox, number_grids=4)

# by_dist_time_speed functions segments the trajectories into clusters based on distance, time and speed. The distance, time and speed limits by the parameters by max_dist_between_adj_points, max_time_between_adj_points, max_speed_between_adj_points respectively. The column tid_part is added, it indicates the segment to which the point belongs to.

segmentation.by_dist_time_speed(df_move, max_dist_between_adj_points=5000, 
                                max_time_between_adj_points=800,max_speed_between_adj_points=60.0)
df_move.head()

# by_speed function segments the trajectories into clusters based on speed. The speed limit is defined by the parameter max_speed_between_adj_points. The column tid_speed is added, it indicates the segment to  which the point belongs to.

segmentation.by_speed(df_move, max_speed_between_adj_points=70.0)
df_move.head()

# by_time function segments the trajectories into clusters based on time. The time limit is defined by the parameter max_time_between_adj_points. The column tid_time is added, it indicates the segment to  which the point belongs to.

segmentation.by_time(df_move, max_time_between_adj_points = 1000)
df_move.head()

# segment_traj_by_max_dist function segments the trajectories into clusters based on distance. The distance limit is defined by the parameter max_dist_between_adj_points. The column tid_dist is added, it indicates the segment to which the point belongs to.

segmentation.by_max_dist(df_move, max_dist_between_adj_points = 4000)
df_move.head()

# ## Stay point detection 

# A stay point is location where a moving object has stayed for a while within a certain distance threshold. A stay point could stand different places such: a restaurant, a school, a work place.

# Importing the module:

from pymove import stay_point_detection

# stay_point_detection function converts the time data into a cyclical format. The columns hour_sin and hour_cos are added to the dataframe.

stay_point_detection.create_update_datetime_in_format_cyclical(df_move)

df_move.head()

# create_or_update_move_stop_by_dist_time function creates or updates the stay points of the trajectories, based on distance and time metrics. The column segment_stop is added to the dataframe, it indicates the trajectory segment to  which the point belongs to. The column stop is also added, it indicates is the point represents a stop, a place where the object was stationary.

stay_point_detection.create_or_update_move_stop_by_dist_time(df_move, dist_radius=40, time_radius=1000)

df_move.head()

# create_update_move_and_stop_by_radius function creates or updates the stay points of the trajectories, based on distance. The column situation is also added, it indicates if the point represents a stop point or a moving point.

stay_point_detection.create_update_move_and_stop_by_radius(df_move, radius=2)

df_move.head()

# ## Compression

# Importing the module:

from pymove import compression

# The function below is used to reduce the size of the trajectory, the stop points are used to make the compression. 

# +
# compression.compress_segment_stop_to_point(df_move)
# -

# ## Map matching
#
# Import module:

from pymove import map_matching

# +
# map_matching.check_time_dist(df_move)
# -



StarcoderdataJupyterScriptsDedupFiltered