# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: 'Python 3.8.3 64-bit (''aiml1'': conda)'
#     metadata:
#       interpreter:
#         hash: 4679b77d4638167d678d58301c2a68afb4de0bda7d7a8bd5f7f4da2854807b37
#     name: 'Python 3.8.3 64-bit (''aiml1'': conda)'
# ---

# # Numpy Intro
#
# ## Reference:
#
# 1. https://www.w3schools.com/python/numpy_intro.asp  
# 2. https://numpy.org/  
# 3. http://www.learningaboutelectronics.com/Articles/How-to-create-an-array-of-random-integers-in-Python-with-numpy.php  
#

# ## Getting started

# +
import numpy as np
arr = np.array([1,2,3,4,5])
print(arr)
print(type(arr))

print(np.__version__)
# -

# ## Using tuple to create array

arr = np.array((1,2,3,4,5,6,7))
print(arr)

# ## Dimensions

# +
# 0 dimensional, with single value
arr = np.array(42)
print(arr)

# check number of dimensions
print("# of dimensions : ", arr.ndim) 

# +
# 1 dimensional, with one array
arr = np.array([7,14,21,28,35,42])
print(arr)

# check number of dimensions
print("# of dimensions : ", arr.ndim) 

# +
# 2 dimensional, an array that has 1D arrays as elements
arr = np.array([
        [1,2],
        [3,4]
    ])
print(arr)

# check number of dimensions
print("# of dimensions : ", arr.ndim) 

# +
# 3 dimensional, an array that has 2D arrays as its elements
arr = np.array([
    [
        [1,2],
        [3,4]
    ],
    [
        [5,6],
        [7,9]
    ]
])
print(arr)

# check number of dimensions
print("# of dimensions : ", arr.ndim) 
# -

# ## Creating higher dimensional array using numpy

arr = np.array([1,2,3,4], ndmin=5)
print(arr)
# check number of dimensions
print("# of dimensions : ", arr.ndim) 

arr = np.array([
    [[1,2,3,4],[5,6,7,8]],
    [[11,12,13,14],[15,16,17,18]]
], ndmin=5)
print(arr)
# check number of dimensions
print("# of dimensions : ", arr.ndim) 

# ## Array indexing
# Indexing is used to access the elemets in the array

# +
print('accessing the 1st level')
print('-'*20)
print(arr[0])
print('-'*20)

print('accessing the 2nd level')
print('-'*20)
print(arr[0][0])
print('-'*20)

print('accessing the 3rd level')
print('-'*20)
print(arr[0][0][0])
print('-'*20)

print('accessing the 4th level')
print('-'*20)
print(arr[0][0][1][1])
print('-'*20)

print('accessing the 5th level')
print('-'*20)
print(arr[0][0][1][1][3])


# -

# ## Array can be accessed by passing an array of indices
#
# arr\[l,m,n,o\] => returns the 4D array element, if its present

# +
print('accessing the 1st level')
print('-'*20)
print(arr[0])
print('-'*20)

print('accessing the 2nd level')
print('-'*20)
print(arr[0,0])
print('-'*20)

print('accessing the 3rd level')
print('-'*20)
print(arr[0,0,0])
print('-'*20)

print('accessing the 4th level')
print('-'*20)
print(arr[0,0,1,1])
print('-'*20)

print('accessing the 5th level')
print('-'*20)
print(arr[0,0,1,1,3])
# -

# ## Negative indexing

# +
print('accessing the 3rd level, last element')
print('-'*20)
print(arr[0,0,-1])
print('-'*20)

print('accessing the 4th level, last element')
print('-'*20)
print(arr[0,0,-1,-1])
print('-'*20)

print('accessing the 5th level, second last element of the first element at 4th level')
print('-'*20)
print(arr[0,0,-1,-2,-2])
print('-'*20)
# -

# ## Array slicing
#
# Slicing helps in taking elements from one index to another index
#
# e.g.
#
# arr\[start:end\]  
# arr\[start:end:step\]  
# arr\[start:end,start,:end\] etc.
#

# +
# 1D array
arr = np.random.randint(1,100,10)
print(arr)
print("# of dims",arr.ndim)
print("-"*20)

print("printing elements from 2nd to 4th")
print(arr[1:4])

print("printing elements from 4nd to end")
print(arr[4:])

print("printing elements till 7th")
print(arr[:7])
# -

# ## Negative slicing

# +
arr = np.random.randint(1,100,10)
print(arr)
print("# of dims",arr.ndim)
print("-"*20)


print("printing 3rd last elements till 2nd last elemet")
print(arr[-3:-1]) 
print("-"*20)

print("printing 6th last elements till the last elemet")
print(arr[-6:]) 
# -

# ## using step

# +
arr = np.random.randint(1,100,10)
print(arr)
print("# of dims",arr.ndim)
print("-"*20)

print("elements at the 2nd index steps")
print(arr[::2])
print("-"*20)

print("elements at the 2nd index steps from 1st till 5th element")
print(arr[:5:2])
print("-"*20)

print("elements at even 2nd indices steps from 1st till 5th element")
print(arr[1:5:2])
print("-"*20)


# +
# 2D array
arr = np.random.randint(1,100, size=(2,5))
print(arr)
print("# of dims",arr.ndim)
print("-"*20)

print("printing elements from index 0 ")
print(arr[0:])
print("# of dims",arr[0:].ndim)
print("-"*20)

print("printing the first element from 2D array ")
print(arr[0:1])
print("# of dims",arr[0:1].ndim)
print("-"*20)

print("printing the second elements from 2D array ")
print(arr[1:])
print("# of dims",arr[1:].ndim)
print("-"*20)

print("printing the second element of the second element from 2D array ")
print(arr[1:])
print("# of dims",arr[1:,1:2].ndim)
print("-"*20)

# +
# 2D array
arr = np.random.randint(1,100, size=(2,5))
print(arr)
print("# of dims",arr.ndim)
print("-"*20)


print("printing the 3rd and 4th elements from all rows of 2D array")
print(arr[0:,2:4])
print("# of dims",arr[0:,2:4].ndim)
print("-"*20)


print("printing the 3rd and 4th elements from the second row of 2D array")
print(arr[1:2,2:3])
print("# of dims",arr[1:2,2:3].ndim)
print("-"*20)


# +
# 3D array
print("3 dimentional array, with 2 rows and 3 elements")
arr = np.random.randint(1,10, size=(3,2,3)) 
print(arr)
print("# of dims",arr.ndim)
print("-"*20)


print("3 dimentional array, with 5 rows and 7 elements")
arr = np.random.randint(1,100, size=(3,5,7)) 
print(arr)
print("# of dims",arr.ndim)
print("-"*20)

# +
print("the first item in the 3D array using simple index gives 1D array")
print(arr[0])
print("# of dims",arr[0].ndim)
print("-"*20)

print("the first item in the 3D array, using slice gives 3D array")
print(arr[0:1])
print("# of dims",arr[0:1].ndim)
print("-"*20)
# -

print("the second element in 3D array")
print(arr[1:2])
print("# of dims",arr[0:1].ndim)
print("-"*20)

print("the 3rd and 4th element from the second element in 3D array")
print(arr[1:2,2:4])
print("# of dims",arr[1:2,2:4].ndim)
print("-"*20)

# +
print("the 6th element from the 3rd and 4th element from the second element in 3D array")
print(arr[1:2,2:4,6:7])
print("# of dims",arr[1:2,2:4, 6:7].ndim)
print("-"*20)

print("the 6th element, from the 5th element from the second element in 3D array")

"""
1. 1:2 idexes the 2nd element of 3D array 
2. 4:5 indexes the 5th element of the array obtained as a result of step 1
3. 5:6 indexes the 6th element of the array obtained as a result of step 2
"""
print(arr[1:2,4:5,5:6]) 
print("# of dims",arr[1:2, 4:5, 5:6].ndim)
print("-"*20)
# -

# ## Data types
#
# Below is a list of all data types in NumPy and the characters used to represent them.
#
#     i - integer
#     b - boolean
#     u - unsigned integer
#     f - float
#     c - complex float
#     m - timedelta
#     M - datetime
#     O - object
#     S - string
#     U - unicode string
#     V - fixed chunk of memory for other type ( void )
#

# +
arr = np.array([1, 2, 3, 4])
print(arr)
print("datatype is : ",arr.dtype) 
print("-"*20)

arr = np.array(["apple", 'banana', 'mango', 'peach'])
print(arr)
print("datatype is : ",arr.dtype) 
print("-"*20)

print("with a defined datatype")
arr = np.array([1,2,3,4], dtype="S")
print(arr)
print("datatype is : ",arr.dtype) 
print("-"*20)
# -

# For i, u, f, S and U we can define size as well.

# +
arr = np.array([1, 2, 3, 4], dtype='i4')
print(arr)
print(arr.dtype) 
print("-"*20)

arr = np.array([1, 2, 3, 4], dtype='i8')
print(arr)
print(arr.dtype) 
print("-"*20)
# -

# ## Converting datatype of existing array

# +
arr = np.array([1.1, 2.1, 3.1])
print('Original array : ',arr)
print(arr.dtype) 
print('-'*20)

newarr = arr.astype('i')

print('New array',newarr)
print(newarr.dtype) 
print('-'*20)

# +
arr = np.array([1.1, 2.1, 3.1])
print('Original array : ',arr)
print(arr.dtype) 
print('-'*20)

newarr = arr.astype(int)

print('New array',newarr)
print(newarr.dtype) 
print('-'*20)

# +
arr = np.array([1.1, 2.1, 0.0])
print('Original array : ',arr)
print(arr.dtype) 
print('-'*20)

newarr = arr.astype(bool)

print('New array',newarr)
print(newarr.dtype) 
print('-'*20)

# +
arr = np.array([1.1, 2.1, 0.0])
print('Original array : ',arr)
print(arr.dtype) 
print('-'*20)

newarr = arr.astype(str)

print('New array',newarr)
print(newarr.dtype) 
print('-'*20)
# -

# ## Copy vs View

# +
print("changing the element at index 0 of the original array has no effect on copy")
print("-"*20)

arr = np.array([1, 2, 3, 4, 5])
print("arr before changing the element :",arr)
x = arr.copy()
arr[0] = 42

print("arr :",arr)
print("x   :",x) 

# +
print("changing the element at index 0 of the original array has effect on view")
print("-"*20)

arr = np.array([1, 2, 3, 4, 5])
print("arr before changing the element :",arr)
x = arr.view()
arr[0] = 42

print("arr :",arr)
print("x   :",x) 
# -

# ### Every NumPy array has the attribute `base` that returns None if the array owns the data.
#
#

# +
arr = np.array([1, 2, 3, 4, 5])

x = arr.copy()
y = arr.view()

print("base for copy : ",x.base)
print("base for view : ",y.base , ", => returns the original array") 
# -

# ## Shape

print("3 dimentional array, with 5 rows and 7 elements")
arr = np.random.randint(1,100, size=(3,5,7)) 
print(arr)
print("# of dims",arr.ndim)
print("Shape of the array is : ", arr.shape)
print("Size of the array is : ", arr.size)
print("-"*20)

# ## Reshape

# +
print("3 dimentional array, with 4 rows and 2 elements")
arr = np.random.randint(1,100, size=(3,4,2)) 
print(arr)
print("# of dims",arr.ndim)
print("Shape of the array is : ", arr.shape)
print("Size of the array is : ", arr.size)
print("-"*20)

print("Reshaping array to 4 dimentional array, with 2 rows of 3, 2D arrays")
"""
4th dimension has 2 3D arrays
3rd dimesion has 3 2D arrays
2nd dimension has 2 arrays with 2 elements each
1st dimention has 2 elements 
"""
newarr = arr.reshape(2,3,2,2) # should be equal to dimension
print(newarr)
print("# of dims in newarr",newarr.ndim)
print("Shape of the newarr is : ", newarr.shape) # why this is not changing?
print("Size of the newarr is : ", newarr.size)
print("-"*20)

# +
# can reshape only to same size array

# Unknown dimension
# Note: We can not pass -1 to more than one dimension.

"""
4th dimension has 6 3D arrays
3rd dimesion has 2 2D arrays
2nd dimension has 2 arrays with 1 elements each
1st dimention has 1 elements 
"""
newarr = arr.reshape(-1,2,2,1) # should be equal to dimension, 4 in this case as we want 4D array
print(newarr)
print("# of dims in newarr",newarr.ndim)
print("Shape of the newarr is : ", newarr.shape)
print("Size of the newarr is : ", newarr.size)
print("-"*20)

# -

# ## Flattening the arrays

flatarr = newarr.reshape(-1)
print(flatarr)
print("# of dims in flatarr",flatarr.ndim)
print("Shape of the flatarr is : ", flatarr.shape)
print("Size of the flatarr is : ", flatarr.size)
print("-"*20)


# **Note:** There are a lot of functions for changing the shapes of arrays in numpy `flatten`, `ravel` and also for rearranging the elements `rot90`, `flip`, `fliplr`, `flipud` etc. These fall under Intermediate to Advanced section of numpy.

print("flatten", newarr.flatten())
print("ravel  ", newarr.ravel())

# ## Iterating

# 1D array
arr = np.random.randint(1,100,10)
for i in arr:
    print(i)

# 2D array
arr = np.random.randint(1,100, size=(3,5))
for k,v in enumerate(arr):
    print("row-"+str(k)+" : ",v)

# 3D array
arr = np.random.randint(1,100, size=(3,5,2))
for i,x in enumerate(arr):
    print("row-"+str(i)+" : \n",x)
    print("-"*20)
    for j,y in enumerate(x):
        print("row-"+str(i)+","+str(j)+" : ",y)
    print("="*20)

# 4D array
arr = np.random.randint(1,100, size=(2,1,3,4))
for i,x in enumerate(arr):
    print("4D row-"+str(i)+" : \n",x)
    print("-"*20)
    for j,y in enumerate(x):
        print("3D row-"+str(i)+","+str(j)+" : \n",y)
        for k,z in enumerate(y):
            print("2D row-"+str(i)+","+str(j)+","+str(k)+" : ",z)
            for l,a in enumerate(z):
                print("1D row-"+str(i)+","+str(j)+","+str(k)+","+str(l)+" : ",a)
            print("="*20)

# ## Iterating Arrays Using `nditer()`
#
# The function nditer() is a helping function that can be used from very basic to very advanced iterations. 

# +
# 3D array
arr = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])

# prints each element in the nd array
for x in np.nditer(arr):
  print(x) 
# -

# ## Iterating Array With Different Data Types
#
# We can use `op_dtypes` argument and pass it the expected datatype to change the datatype of elements while iterating.
#
# NumPy does not change the data type of the element in-place (where the element is in array) so it needs some other space to perform this action, that extra space is called buffer, and in order to enable it in `nditer()` we pass `flags=['buffered']`.

# +
# 3D array
arr = np.random.randint(1,100, size=(2,2,2))

for i in np.nditer(arr, op_dtypes=['S'], flags=['buffered']):
    print(i)
# -

# ## nditer step size
#
#

# +
# 3D array
arr = np.random.randint(1,100, size=(5,3,8))

print(arr)
print('-'*20)
print(arr[1:5:2, fc00:e968:6179::de52:7100,3::4])

"""
1. identify the rows from 3D array, 1:5:2 => every other row starting from 2nd element, (index 1 and 3)
2. get row at index 0 and 2 from array at index 1 and 3 from prev step
3. get every 4th element, starting form index 3 from prev step result
"""
for x in np.nditer(arr[1:5:2, fc00:e968:6179::de52:7100,3::4]):
  print(x) 

# -

# ## Enumerated Iteration Using `ndenumerate()`

arr = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])
for idx, x in np.ndenumerate(arr):
  print(idx, x) 

arr  = np.random.randint(1,100,size=(4,5,2))
for idx,x in np.ndenumerate(arr):
    print(idx,x)

# ## Joining NumPy Arrays
#
# Joining means putting contents of two or more arrays in a single array.  
# In SQL we join tables based on a key, whereas in NumPy we join arrays by axes.  
# axis = 0 => rows  
# axis = 1 => cols  
#
#     - concatinate
#     - stack
#         - hstack
#         - vstack
#         - dstack
#     
#

# +
# 1D concatination
arr1 = np.random.randint(1,100,size=(10))
arr2 = np.random.randint(1,100,size=(10))

arr = np.concatenate((arr1,arr2))

print(arr1)
print(arr2)
print(arr)

# +
# 2D concatination
arr1 = np.random.randint(1,100,size=(3,10))
arr2 = np.random.randint(1,100,size=(3,1))# just 1 col but same number of rows
# arr2 = np.random.randint(1,100,size=(4,1))# just 1 col but different number of rows, this does not work in case of axis 1, will work for axis = 0

# with axis = 1
arr = np.concatenate((arr1,arr2), axis=1)

print(arr1)
print("-"*20)
print(arr2)
print("-"*20)
print(arr)
# -

# ## Stack

# +
arr1 = np.random.randint(1,50,size=(3,7))
arr2 = np.random.randint(1,50,size=(3,7))

arr = np.stack((arr1,arr2))
print(arr1)
print("-"*20)
print(arr2)
print("-"*20)
print("this puts arr1 on top of arr2 and creates 3D array as arr1 and arr2 are 2D arrays, has 2 rows")
print(arr)
print("# of dims",arr.ndim)
print("shape of dims",arr.shape)

# +
arr1 = np.random.randint(1,50,size=(3,7))
arr2 = np.random.randint(1,50,size=(3,7))

arr = np.stack((arr1,arr2), axis=1)
print(arr1)
print("-"*20)
print(arr2)
print("-"*20)
print("this puts row at index i from arr1 and arr2 on top of each other and creates 3D array as arr1 and arr2 are 2D arrays, has 3 rows now")
print(arr)
print("# of dims",arr.ndim)
print("shape of dims",arr.shape)

# +
# hstack
arr1 = np.random.randint(1,50,size=(3,7))
arr2 = np.random.randint(1,50,size=(3,7))

arr = np.hstack((arr1,arr2))
print(arr1)
print("-"*20)
print(arr2)
print("-"*20)
print("concatinates corresponding row from arr1 and arr2 at index i, changes number of columns ")
print(arr)
print("# of dims",arr.ndim)
print("shape of dims",arr.shape)

# +
# vstack
arr1 = np.random.randint(1,50,size=(3,7))
arr2 = np.random.randint(1,50,size=(3,7))

arr = np.vstack((arr1,arr2))
print(arr1)
print("-"*20)
print(arr2)
print("-"*20)
print("stacks one array on top of another array, changes number of rows")
print(arr)
print("# of dims",arr.ndim)
print("shape of dims",arr.shape)

# +
# dstack
arr1 = np.random.randint(1,50,size=(2,7))
arr2 = np.random.randint(1,50,size=(2,7))
arr3 = np.random.randint(1,50,size=(2,7))

arr = np.dstack((arr1,arr2,arr3))
print(arr1)
print("-"*20)
print(arr2)
print("-"*20)
print("creates rows in 3D array = # of rows in arr1,arr2,arr3, 2 in this case")
print("creates colums = # of arrays, arr1,arr2,arr3 = 3")
print("creates rows in each 2D array = # of cols in arr1,arr2,arr3 = 7")
print(arr)
print("# of dims",arr.ndim)
print("shape of dims",arr.shape)

# -

# ## Splitting NumPy Arrays
#
# Splitting is reverse of joining
#
# we use `array_split` method as `split` method does not adjust if the number of elemets dont match
#

# split array into 3 arrays
arr = np.random.randint(1,50,size=(10))
newarr = np.array_split(arr, 3)
print(arr)
print("-"*20)
print()
print(newarr) 

# split array into 4 arrays
arr = np.random.randint(1,50,size=(6))
newarr = np.array_split(arr, 4)
print(arr)
print("-"*20)
print(newarr) 

# +
# 2D array, split array into 3 arrays, col axis, so rows are adjusted
arr = np.random.randint(1,50,size=(5,3))
newarr = np.array_split(arr, 3)
print(arr)
print("-"*20)

# array will be 2D, number of columns will not change number of rows will be adjuted if required
for k,val in np.ndenumerate(newarr):
    print(k,"\n",val)
# -

# 2D array, split array into 2 arrays, row axis, so cols are adjusted
arr = np.random.randint(1,50,size=(4,3))
newarr = np.array_split(arr, 2, axis=1)
print(arr)
print("-"*20)
print(newarr)
print("-"*20)
# array will be 2D, number of rows will not change number of cols will be adjuted if required
# first roe has 2 cols
# second row has 1 col

# +
# hsplit

arr = np.random.randint(1,50,size=(2,8))
newarr = np.hsplit(arr, 4)
print(arr)
print("-"*20)
print(newarr)
print("-"*20)
# need 4 arrays
# first 2 cols for each row make the first array
# 3rd and 4th cols for each row make the 2nd array
# 5th and 6th cols for each row make the 3rd array
# 7th and 8th cols for each row make the 4th array
# -

# **`Note:`** Similar alternates to `vstack()` and `dstack()` are available as `vsplit()` and `dsplit()`.

# ## NumPy Searching Arrays
#
# Searching array returns the indices of the result  
# To search an array, use the `where()` method.

arr = np.array([1, 2, 3, 4, 5, 4, 4])
x = np.where(arr == 4)
print(x) 

# +
# every third element till 50
arr = np.arange(1,50,3)
print(arr)

# index for every even number in the above list
x = np.where(arr%2 == 0)
print(x)

# +
# every fifth element till 50
arr = np.arange(1,50,5)
print(arr)

# index for every odd number in the above list
x = np.where(arr%2 == 1)
print(x)
# -

# ### search sorted

arr = np.arange(2,50,2)
print(arr)
# search single value in sorted order
x = np.searchsorted(arr,14)
print(x)

arr = np.arange(2,50,2)
print(arr)
# search single value in sorted order from right side
x = np.searchsorted(arr, 14, side='right') # this one has issue
print('Index of 14 is : ',x)
print("Element at "+str(x)+" is : ",arr[x])

arr = np.arange(2,50,2)
print(arr)
# search single value in sorted order from right side
x = np.searchsorted(arr, [14,48,40])
print(x)

# ## Sorting Arrays
#
# putting elements in ordered sequence

# +
# number
arr = np.array([3, 2, 0, 1])
print(arr,"\n", np.sort(arr),'\n----\n') 

# String
arr = np.array(['banana', 'cherry', 'apple'])
print(arr, "\n", np.sort(arr),"\n----\n") 

# boolean
arr = np.array([True, False, True])
print(arr, "\n", np.sort(arr),"\n----\n") 

# 2D array
arr = np.array([[3, 2, 4], [5, 0, 1]])
print(arr, "\n----\n", np.sort(arr)) 
# -

# reversed sort
arr = np.array([3, 2, 45, 0, 1])
print(arr, "\n----\n", np.sort(arr)[::-1]) 
print('-'*20)
arr = np.array([[3, 2, 4], [5, 0, 1]])
print(arr, "\n----\n", np.sort(arr)[:,::-1]) 

# ## Filter arrays
#
# Getting some elements out of an existing array and creating a new array out of them is called filtering.
#
# In NumPy, you filter an array using a boolean index list.
#
#
#
# A boolean index list is a list of booleans corresponding to indexes in the array.
#
# If the value at an index is True that element is contained in the filtered array, if the value at that index is False that element is excluded from the filtered array.

arr = np.array([41, 42, 43, 44])
# print item at index 0 and 2 
x = [True, False, True, False]
newarr = arr[x]
print(newarr)

# +
# print elements higher than 42
arr = np.array([41, 42, 43, 44])

# create empty filter array
filter_arr = []

for i in arr:
    if i >42:
        filter_arr.append(True)
    else:
        filter_arr.append(False)

# pass the filter_array to actual array
newarr = arr[filter_arr]

print(filter_arr)
print(newarr)



# +
# Create a filter array that will return only even elements from the original array

arr = np.arange(1,21)

# create empty filter array
filter_arr = []

for i in arr:
    if i %2 == 0:
        filter_arr.append(True)
    else:
        filter_arr.append(False)

# pass the filter_array to actual array
newarr = arr[filter_arr]

print(filter_arr)
print(newarr)

# -

# # Creating Filter Directly From Array

# Create a filter array that will return only values higher than 42:
arr = np.array([41, 42, 43, 44])
filter_arr = arr > 42
newarr = arr[filter_arr]
print(filter_arr)
print(newarr) 

# Create a filter array that will return only even elements from the original array:
arr = np.array([1, 2, 3, 4, 5, 6, 7])
filtered_arr = arr %2 ==0
newarr = arr[filtered_arr]
print(filter_arr)
print(newarr) 



StarcoderdataJupyterScriptsDedupFiltered