We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Remove pandas as a dependency
pandas is only used in the IndianSyllabifier and only for reading a csv file before its data in converted to a numpy array. pandas is also the largest external dependency (e.g. slowest to install in travis-ci). The csv reading and conversion to numpy can be done with the standard libraries, spec. ```csv```
Remove pandas as a dependency
pandas is only used in the IndianSyllabifier and only for reading a csv file before its data in converted to a numpy array. pandas is also the largest external dependency (e.g. slowest to install in travis-ci). The csv reading and conversion to numpy can be done with the standard libraries, spec. ```csv```

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `cltk/stem/sanskrit/indian_syllabifier.py`
Content:
```
1 """Every phonetic of every language is given similar positions in the vectors. Therefore transliterations
2 happen when each offset is calculated relative to the ranges of the languages specified.
3 Every phonetic has a dedicated phonetic vector which describes all the facets of the character, whether it is
4 a vowel or a consonant whe ther it has a halanta, etc.
5 
6 Source: https://github.com/anoopkunchukuttan/indic_nlp_library/blob/master/src/indicnlp/script/indic_scripts.py
7 """
8 
9 import os
10 
11 try:
12     import numpy as np
13     import pandas as pd
14 except ImportError:
15     print('"pandas" and "numpy" libraries not installed.')
16     raise
17 
18 __author__ = ['Anoop Kunchukuttan']
19 __license__ = 'GPLv3'
20 
21 
22 # Indexes into the phonetic vector
23 PVIDX_BT_VOWEL = 0
24 PVIDX_BT_CONSONANT = 1
25 PVIDX_BT_NUKTA = 2
26 PVIDX_BT_HALANT = 3
27 PVIDX_BT_ANUSVAAR = 4
28 PVIDX_BT_MISC = 5
29 PVIDX_BT_S = PVIDX_BT_VOWEL
30 PVIDX_BT_E = PVIDX_BT_MISC + 1
31 
32 PVIDX_VSTAT_DEP = 12
33 
34 LC_TA = 'ta'
35 
36 LANGUAGE_NAME_TO_CODE = {'hindi': 'hi', 'sanskrit': 'sa', 'punjabi': 'pa', 'gujarati': 'gu', 'oriya': 'or',
37                          'tamil': 'ta', 'telegu': 'te', 'kannada': 'kn', 'malayalam': 'ml', 'sinhalese': 'si',
38                          'marathi': 'mr', 'konkan': 'kk', 'nepali': 'ne', 'sindhi': 'sd', 'bengali': 'bn',
39                          'assamese': 'as'}
40 
41 
42 # The phonetics of every script exist in the ranges of the dictionary mentioned below
43 SCRIPT_RANGES = {
44     'pa': [0x0a00, 0x0a7f],
45     'gu': [0x0a80, 0x0aff],
46     'or': [0x0b00, 0x0b7f],
47     'ta': [0x0b80, 0x0bff],
48     'te': [0x0c00, 0x0c7f],
49     'kn': [0x0c80, 0x0cff],
50     'ml': [0x0d00, 0x0d7f],
51     'si': [0x0d80, 0x0dff],
52     'hi': [0x0900, 0x097f],
53     'mr': [0x0900, 0x097f],
54     'kk': [0x0900, 0x097f],
55     'sa': [0x0900, 0x097f],
56     'ne': [0x0900, 0x097f],
57     'sd': [0x0900, 0x097f],
58     'bn': [0x0980, 0x09ff],
59     'as': [0x0980, 0x09ff],
60 }
61 
62 COORDINATED_RANGE_START_INCLUSIVE = 0
63 COORDINATED_RANGE_END_INCLUSIVE = 0x6f
64 
65 PV_PROP_RANGES = dict(basic_type=[0, 6], vowel_length=[6, 8], vowel_strength=[8, 11], vowel_status=[11, 13],
66                       consonant_type=[13, 18], articulation_place=[18, 23], aspiration=[23, 25], voicing=[25, 27],
67                       nasalization=[27, 29], vowel_horizontal=[29, 32], vowel_vertical=[32, 36],
68                       vowel_roundness=[36, 38])
69 
70 PHONETIC_VECTOR_START_OFFSET = 6
71 
72 
73 class Syllabifier:
74     """Class for syllabalizing Indian language words."""
75 
76     def __init__(self, lang_name):
77         """Setup values."""
78 
79         self.lang_name = lang_name
80         assert self.lang_name in LANGUAGE_NAME_TO_CODE.keys(), 'Language not available'
81         self.lang = LANGUAGE_NAME_TO_CODE[lang_name]
82 
83         assert self.lang in SCRIPT_RANGES.keys()
84 
85         self.all_phonetic_data, self.tamil_phonetic_data, self.all_phonetic_vectors, self.tamil_phonetic_vectors, self.phonetic_vector_length = self.get_lang_data()
86 
87     def get_lang_data(self):
88         """Define and call data for future use. Initializes and defines all
89         variables which define the phonetic vectors.
90         """
91 
92         root = os.path.expanduser('~')
93         csv_dir_path = os.path.join(root, 'cltk_data/sanskrit/model/sanskrit_models_cltk/phonetics')
94 
95         all_phonetic_csv = os.path.join(csv_dir_path, 'all_script_phonetic_data.csv')
96         all_phonetic_data = pd.read_csv(all_phonetic_csv, encoding='utf-8')
97         tamil_csv = os.path.join(csv_dir_path, 'tamil_script_phonetic_data.csv')
98         tamil_phonetic_data = pd.read_csv(tamil_csv, encoding='utf-8')
99 
100         all_phonetic_vectors = all_phonetic_data.ix[:, PHONETIC_VECTOR_START_OFFSET:].values
101         tamil_phonetic_vectors = tamil_phonetic_data.ix[:, PHONETIC_VECTOR_START_OFFSET:].values
102 
103         phonetic_vector_length = all_phonetic_vectors.shape[1]
104 
105         return all_phonetic_data, tamil_phonetic_data, all_phonetic_vectors, tamil_phonetic_vectors, phonetic_vector_length
106 
107     @staticmethod
108     def in_coordinated_range_offset(c_offset):
109         """Applicable to Brahmi derived Indic scripts. Used to determine 
110         whether offset is of a  alphabetic character or not.
111         """
112         return COORDINATED_RANGE_START_INCLUSIVE <= c_offset <= COORDINATED_RANGE_END_INCLUSIVE
113 
114     def get_offset(self, c, lang):
115         """Gets the offset; that is the relative position in the range of the
116         specified language.
117         """
118         return ord(c) - SCRIPT_RANGES[lang][0]
119 
120     def invalid_vector(self):
121         """Returns an zero array of length 38"""
122         return np.array([0] * self.phonetic_vector_length)
123 
124     def get_phonetic_info(self, lang):
125         """For a specified language (lang), it returns the matrix and the vecto
126          containing specifications of the characters.
127          """
128         phonetic_data = self.all_phonetic_data if lang != LC_TA else self.tamil_phonetic_data
129         phonetic_vectors = self.all_phonetic_vectors if lang != LC_TA else self.tamil_phonetic_vectors
130 
131         return phonetic_data, phonetic_vectors
132 
133     def get_phonetic_feature_vector(self, c, lang):
134         """For a given character in a language, it gathers all the information related to it
135           (eg: whether fricative, plosive,etc)"""
136 
137         offset = self.get_offset(c, lang)
138         if not self.in_coordinated_range_offset(offset):
139             return self.invalid_vector()
140 
141         phonetic_data, phonetic_vectors = self.get_phonetic_info(lang)
142 
143         if phonetic_data.ix[offset, 'Valid Vector Representation'] == 0:
144             return self.invalid_vector()
145 
146         return phonetic_vectors[offset]
147 
148     def get_property_vector(self, v, prop_name):
149         """Returns the part of the vector corresponding to the required property"""
150         return v[PV_PROP_RANGES[prop_name][0]:PV_PROP_RANGES[prop_name][1]]
151 
152 
153     def is_consonant(self, v):
154         """Checks the property of the character (of being a consonant)
155         selected against its phonetic vector.
156         """
157         return v[PVIDX_BT_CONSONANT] == 1
158 
159 
160     def is_misc(self,v):
161         """Checks the property of the character (of being miscellenous)
162         selected against its phonetic vector.
163         """
164         return v[PVIDX_BT_MISC] == 1
165 
166     def is_valid(self, v):
167         """Checks if the character entered is valid, by checking against the
168         phonetic vector. At least 1 of the 38 properties have to be
169         satisfied for a valid vector.
170         """
171         return np.sum(v) > 0
172 
173     def is_vowel(self, v):
174         """Checks the property of the character (of being a vowel) selected against its phonetic vector
175             """
176         return v[PVIDX_BT_VOWEL] == 1
177 
178     def is_anusvaar(self, v):
179         """Checks the property of the character (of having an anusvaar)
180         selected against its phonetic vector.
181         """
182         return v[PVIDX_BT_ANUSVAAR] == 1
183 
184     def is_plosive(self, v):
185         """Checks the property of the character (of being a plosive
186         character) selected against its phonetic vector.
187          """
188         return self.is_consonant(v) and self.get_property_vector(v, 'consonant_type')[0] == 1
189 
190     def is_nukta(self,v):
191         """Checks the property of the character (of having a nukta) selected
192         against its phonetic vector.
193         """
194         return v[PVIDX_BT_NUKTA] == 1
195 
196     def is_dependent_vowel(self, v):
197         """Checks the property of the character (if it is a dependent
198         vowel) selected against its phonetic vector.
199         """
200         return self.is_vowel(v) and v[PVIDX_VSTAT_DEP] == 1
201 
202     def orthographic_syllabify(self, word):
203         """Main syllablic function."""
204         p_vectors = [self.get_phonetic_feature_vector(c, self.lang) for c in word]
205 
206         syllables = []
207 
208         for i in range(len(word)):
209             v = p_vectors[i]
210 
211             syllables.append(word[i])
212 
213             if i + 1 < len(word) and (not self.is_valid(p_vectors[i + 1]) or self.is_misc(p_vectors[i + 1])):
214                 syllables.append(u' ')
215 
216             elif not self.is_valid(v) or self.is_misc(v):
217                 syllables.append(u' ')
218 
219             elif self.is_vowel(v):
220 
221                 anu_nonplos = (i + 2 < len(word) and
222                                self.is_anusvaar(p_vectors[i + 1]) and
223                                not self.is_plosive(p_vectors[i + 2])
224                                )
225 
226                 anu_eow = (i + 2 == len(word) and
227                            self.is_anusvaar(p_vectors[i + 1]))
228 
229                 if not (anu_nonplos or anu_eow):
230                     syllables.append(u' ')
231 
232             elif i + 1 < len(word) and (self.is_consonant(v) or self.is_nukta(v)):
233                 if self.is_consonant(p_vectors[i + 1]):
234                     syllables.append(u' ')
235                 elif self.is_vowel(p_vectors[i + 1]) and not self.is_dependent_vowel(p_vectors[i + 1]):
236                     syllables.append(u' ')
237                 elif self.is_anusvaar(p_vectors[i + 1]):
238                     anu_nonplos = (i + 2 < len(word) and not self.is_plosive(p_vectors[i + 2]))
239 
240                     anu_eow = i + 2 == len(word)
241 
242                     if not (anu_nonplos or anu_eow):
243                         syllables.append(u' ')
244 
245         return u''.join(syllables).strip().split(u' ')
246 
247 
248 if __name__ == '__main__':
249     syllabifier = Syllabifier('hindi')
250     current = syllabifier.orthographic_syllabify('नमस्ते')
251     print(current)
252
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/cltk/stem/sanskrit/indian_syllabifier.py b/cltk/stem/sanskrit/indian_syllabifier.py
--- a/cltk/stem/sanskrit/indian_syllabifier.py
+++ b/cltk/stem/sanskrit/indian_syllabifier.py
@@ -7,12 +7,12 @@
 """
 
 import os
+import csv
 
 try:
     import numpy as np
-    import pandas as pd
 except ImportError:
-    print('"pandas" and "numpy" libraries not installed.')
+    print('"numpy" is not installed.')
     raise
 
 __author__ = ['Anoop Kunchukuttan']
@@ -93,12 +93,26 @@
         csv_dir_path = os.path.join(root, 'cltk_data/sanskrit/model/sanskrit_models_cltk/phonetics')
 
         all_phonetic_csv = os.path.join(csv_dir_path, 'all_script_phonetic_data.csv')
-        all_phonetic_data = pd.read_csv(all_phonetic_csv, encoding='utf-8')
         tamil_csv = os.path.join(csv_dir_path, 'tamil_script_phonetic_data.csv')
-        tamil_phonetic_data = pd.read_csv(tamil_csv, encoding='utf-8')
 
-        all_phonetic_vectors = all_phonetic_data.ix[:, PHONETIC_VECTOR_START_OFFSET:].values
-        tamil_phonetic_vectors = tamil_phonetic_data.ix[:, PHONETIC_VECTOR_START_OFFSET:].values
+        # Make helper function for this
+        with open(all_phonetic_csv,'r') as f:
+            reader = csv.reader(f, delimiter = ',', quotechar = '"')
+            next(reader, None) # Skip headers
+            all_phonetic_data = [row for row in reader]
+
+        with open(tamil_csv,'r') as f:
+            reader = csv.reader(f, delimiter = ',', quotechar = '"')
+            next(reader, None) # Skip headers
+            # tamil_phonetic_data = [row[PHONETIC_VECTOR_START_OFFSET:] for row in reader]
+            tamil_phonetic_data = [row for row in reader]
+
+        # Handle better?
+        all_phonetic_data = [[int(cell) if cell=='0' or cell=='1' else cell for cell in row] for row in all_phonetic_data]
+        tamil_phonetic_data = [[int(cell) if cell=='0' or cell=='1' else cell for cell in row] for row in tamil_phonetic_data]
+
+        all_phonetic_vectors = np.array([row[PHONETIC_VECTOR_START_OFFSET:] for row in all_phonetic_data])
+        tamil_phonetic_vectors = np.array([row[PHONETIC_VECTOR_START_OFFSET:] for row in tamil_phonetic_data])
 
         phonetic_vector_length = all_phonetic_vectors.shape[1]
 
@@ -106,7 +120,7 @@
 
     @staticmethod
     def in_coordinated_range_offset(c_offset):
-        """Applicable to Brahmi derived Indic scripts. Used to determine 
+        """Applicable to Brahmi derived Indic scripts. Used to determine
         whether offset is of a  alphabetic character or not.
         """
         return COORDINATED_RANGE_START_INCLUSIVE <= c_offset <= COORDINATED_RANGE_END_INCLUSIVE
@@ -140,7 +154,8 @@
 
         phonetic_data, phonetic_vectors = self.get_phonetic_info(lang)
 
-        if phonetic_data.ix[offset, 'Valid Vector Representation'] == 0:
+        # 'Valid Vector Representation' is the [5] column
+        if phonetic_data[offset][5] == 0:
             return self.invalid_vector()
 
         return phonetic_vectors[offset]

{"golden_diff": "diff --git a/cltk/stem/sanskrit/indian_syllabifier.py b/cltk/stem/sanskrit/indian_syllabifier.py\n--- a/cltk/stem/sanskrit/indian_syllabifier.py\n+++ b/cltk/stem/sanskrit/indian_syllabifier.py\n@@ -7,12 +7,12 @@\n \"\"\"\n \n import os\n+import csv\n \n try:\n     import numpy as np\n-    import pandas as pd\n except ImportError:\n-    print('\"pandas\" and \"numpy\" libraries not installed.')\n+    print('\"numpy\" is not installed.')\n     raise\n \n __author__ = ['Anoop Kunchukuttan']\n@@ -93,12 +93,26 @@\n         csv_dir_path = os.path.join(root, 'cltk_data/sanskrit/model/sanskrit_models_cltk/phonetics')\n \n         all_phonetic_csv = os.path.join(csv_dir_path, 'all_script_phonetic_data.csv')\n-        all_phonetic_data = pd.read_csv(all_phonetic_csv, encoding='utf-8')\n         tamil_csv = os.path.join(csv_dir_path, 'tamil_script_phonetic_data.csv')\n-        tamil_phonetic_data = pd.read_csv(tamil_csv, encoding='utf-8')\n \n-        all_phonetic_vectors = all_phonetic_data.ix[:, PHONETIC_VECTOR_START_OFFSET:].values\n-        tamil_phonetic_vectors = tamil_phonetic_data.ix[:, PHONETIC_VECTOR_START_OFFSET:].values\n+        # Make helper function for this\n+        with open(all_phonetic_csv,'r') as f:\n+            reader = csv.reader(f, delimiter = ',', quotechar = '\"')\n+            next(reader, None) # Skip headers\n+            all_phonetic_data = [row for row in reader]\n+\n+        with open(tamil_csv,'r') as f:\n+            reader = csv.reader(f, delimiter = ',', quotechar = '\"')\n+            next(reader, None) # Skip headers\n+            # tamil_phonetic_data = [row[PHONETIC_VECTOR_START_OFFSET:] for row in reader]\n+            tamil_phonetic_data = [row for row in reader]\n+\n+        # Handle better?\n+        all_phonetic_data = [[int(cell) if cell=='0' or cell=='1' else cell for cell in row] for row in all_phonetic_data]\n+        tamil_phonetic_data = [[int(cell) if cell=='0' or cell=='1' else cell for cell in row] for row in tamil_phonetic_data]\n+\n+        all_phonetic_vectors = np.array([row[PHONETIC_VECTOR_START_OFFSET:] for row in all_phonetic_data])\n+        tamil_phonetic_vectors = np.array([row[PHONETIC_VECTOR_START_OFFSET:] for row in tamil_phonetic_data])\n \n         phonetic_vector_length = all_phonetic_vectors.shape[1]\n \n@@ -106,7 +120,7 @@\n \n     @staticmethod\n     def in_coordinated_range_offset(c_offset):\n-        \"\"\"Applicable to Brahmi derived Indic scripts. Used to determine \n+        \"\"\"Applicable to Brahmi derived Indic scripts. Used to determine\n         whether offset is of a  alphabetic character or not.\n         \"\"\"\n         return COORDINATED_RANGE_START_INCLUSIVE <= c_offset <= COORDINATED_RANGE_END_INCLUSIVE\n@@ -140,7 +154,8 @@\n \n         phonetic_data, phonetic_vectors = self.get_phonetic_info(lang)\n \n-        if phonetic_data.ix[offset, 'Valid Vector Representation'] == 0:\n+        # 'Valid Vector Representation' is the [5] column\n+        if phonetic_data[offset][5] == 0:\n             return self.invalid_vector()\n \n         return phonetic_vectors[offset]\n", "issue": "Remove pandas as a dependency\npandas is only used in the IndianSyllabifier and only for reading a csv file before its data in converted to a numpy array. pandas is also the largest external dependency (e.g. slowest to install in travis-ci). The csv reading and conversion to numpy can be done with the standard libraries, spec. ```csv```\nRemove pandas as a dependency\npandas is only used in the IndianSyllabifier and only for reading a csv file before its data in converted to a numpy array. pandas is also the largest external dependency (e.g. slowest to install in travis-ci). The csv reading and conversion to numpy can be done with the standard libraries, spec. ```csv```\n", "before_files": [{"content": "\"\"\"Every phonetic of every language is given similar positions in the vectors. Therefore transliterations\nhappen when each offset is calculated relative to the ranges of the languages specified.\nEvery phonetic has a dedicated phonetic vector which describes all the facets of the character, whether it is\na vowel or a consonant whe ther it has a halanta, etc.\n\nSource: https://github.com/anoopkunchukuttan/indic_nlp_library/blob/master/src/indicnlp/script/indic_scripts.py\n\"\"\"\n\nimport os\n\ntry:\n    import numpy as np\n    import pandas as pd\nexcept ImportError:\n    print('\"pandas\" and \"numpy\" libraries not installed.')\n    raise\n\n__author__ = ['Anoop Kunchukuttan']\n__license__ = 'GPLv3'\n\n\n# Indexes into the phonetic vector\nPVIDX_BT_VOWEL = 0\nPVIDX_BT_CONSONANT = 1\nPVIDX_BT_NUKTA = 2\nPVIDX_BT_HALANT = 3\nPVIDX_BT_ANUSVAAR = 4\nPVIDX_BT_MISC = 5\nPVIDX_BT_S = PVIDX_BT_VOWEL\nPVIDX_BT_E = PVIDX_BT_MISC + 1\n\nPVIDX_VSTAT_DEP = 12\n\nLC_TA = 'ta'\n\nLANGUAGE_NAME_TO_CODE = {'hindi': 'hi', 'sanskrit': 'sa', 'punjabi': 'pa', 'gujarati': 'gu', 'oriya': 'or',\n                         'tamil': 'ta', 'telegu': 'te', 'kannada': 'kn', 'malayalam': 'ml', 'sinhalese': 'si',\n                         'marathi': 'mr', 'konkan': 'kk', 'nepali': 'ne', 'sindhi': 'sd', 'bengali': 'bn',\n                         'assamese': 'as'}\n\n\n# The phonetics of every script exist in the ranges of the dictionary mentioned below\nSCRIPT_RANGES = {\n    'pa': [0x0a00, 0x0a7f],\n    'gu': [0x0a80, 0x0aff],\n    'or': [0x0b00, 0x0b7f],\n    'ta': [0x0b80, 0x0bff],\n    'te': [0x0c00, 0x0c7f],\n    'kn': [0x0c80, 0x0cff],\n    'ml': [0x0d00, 0x0d7f],\n    'si': [0x0d80, 0x0dff],\n    'hi': [0x0900, 0x097f],\n    'mr': [0x0900, 0x097f],\n    'kk': [0x0900, 0x097f],\n    'sa': [0x0900, 0x097f],\n    'ne': [0x0900, 0x097f],\n    'sd': [0x0900, 0x097f],\n    'bn': [0x0980, 0x09ff],\n    'as': [0x0980, 0x09ff],\n}\n\nCOORDINATED_RANGE_START_INCLUSIVE = 0\nCOORDINATED_RANGE_END_INCLUSIVE = 0x6f\n\nPV_PROP_RANGES = dict(basic_type=[0, 6], vowel_length=[6, 8], vowel_strength=[8, 11], vowel_status=[11, 13],\n                      consonant_type=[13, 18], articulation_place=[18, 23], aspiration=[23, 25], voicing=[25, 27],\n                      nasalization=[27, 29], vowel_horizontal=[29, 32], vowel_vertical=[32, 36],\n                      vowel_roundness=[36, 38])\n\nPHONETIC_VECTOR_START_OFFSET = 6\n\n\nclass Syllabifier:\n    \"\"\"Class for syllabalizing Indian language words.\"\"\"\n\n    def __init__(self, lang_name):\n        \"\"\"Setup values.\"\"\"\n\n        self.lang_name = lang_name\n        assert self.lang_name in LANGUAGE_NAME_TO_CODE.keys(), 'Language not available'\n        self.lang = LANGUAGE_NAME_TO_CODE[lang_name]\n\n        assert self.lang in SCRIPT_RANGES.keys()\n\n        self.all_phonetic_data, self.tamil_phonetic_data, self.all_phonetic_vectors, self.tamil_phonetic_vectors, self.phonetic_vector_length = self.get_lang_data()\n\n    def get_lang_data(self):\n        \"\"\"Define and call data for future use. Initializes and defines all\n        variables which define the phonetic vectors.\n        \"\"\"\n\n        root = os.path.expanduser('~')\n        csv_dir_path = os.path.join(root, 'cltk_data/sanskrit/model/sanskrit_models_cltk/phonetics')\n\n        all_phonetic_csv = os.path.join(csv_dir_path, 'all_script_phonetic_data.csv')\n        all_phonetic_data = pd.read_csv(all_phonetic_csv, encoding='utf-8')\n        tamil_csv = os.path.join(csv_dir_path, 'tamil_script_phonetic_data.csv')\n        tamil_phonetic_data = pd.read_csv(tamil_csv, encoding='utf-8')\n\n        all_phonetic_vectors = all_phonetic_data.ix[:, PHONETIC_VECTOR_START_OFFSET:].values\n        tamil_phonetic_vectors = tamil_phonetic_data.ix[:, PHONETIC_VECTOR_START_OFFSET:].values\n\n        phonetic_vector_length = all_phonetic_vectors.shape[1]\n\n        return all_phonetic_data, tamil_phonetic_data, all_phonetic_vectors, tamil_phonetic_vectors, phonetic_vector_length\n\n    @staticmethod\n    def in_coordinated_range_offset(c_offset):\n        \"\"\"Applicable to Brahmi derived Indic scripts. Used to determine \n        whether offset is of a  alphabetic character or not.\n        \"\"\"\n        return COORDINATED_RANGE_START_INCLUSIVE <= c_offset <= COORDINATED_RANGE_END_INCLUSIVE\n\n    def get_offset(self, c, lang):\n        \"\"\"Gets the offset; that is the relative position in the range of the\n        specified language.\n        \"\"\"\n        return ord(c) - SCRIPT_RANGES[lang][0]\n\n    def invalid_vector(self):\n        \"\"\"Returns an zero array of length 38\"\"\"\n        return np.array([0] * self.phonetic_vector_length)\n\n    def get_phonetic_info(self, lang):\n        \"\"\"For a specified language (lang), it returns the matrix and the vecto\n         containing specifications of the characters.\n         \"\"\"\n        phonetic_data = self.all_phonetic_data if lang != LC_TA else self.tamil_phonetic_data\n        phonetic_vectors = self.all_phonetic_vectors if lang != LC_TA else self.tamil_phonetic_vectors\n\n        return phonetic_data, phonetic_vectors\n\n    def get_phonetic_feature_vector(self, c, lang):\n        \"\"\"For a given character in a language, it gathers all the information related to it\n          (eg: whether fricative, plosive,etc)\"\"\"\n\n        offset = self.get_offset(c, lang)\n        if not self.in_coordinated_range_offset(offset):\n            return self.invalid_vector()\n\n        phonetic_data, phonetic_vectors = self.get_phonetic_info(lang)\n\n        if phonetic_data.ix[offset, 'Valid Vector Representation'] == 0:\n            return self.invalid_vector()\n\n        return phonetic_vectors[offset]\n\n    def get_property_vector(self, v, prop_name):\n        \"\"\"Returns the part of the vector corresponding to the required property\"\"\"\n        return v[PV_PROP_RANGES[prop_name][0]:PV_PROP_RANGES[prop_name][1]]\n\n\n    def is_consonant(self, v):\n        \"\"\"Checks the property of the character (of being a consonant)\n        selected against its phonetic vector.\n        \"\"\"\n        return v[PVIDX_BT_CONSONANT] == 1\n\n\n    def is_misc(self,v):\n        \"\"\"Checks the property of the character (of being miscellenous)\n        selected against its phonetic vector.\n        \"\"\"\n        return v[PVIDX_BT_MISC] == 1\n\n    def is_valid(self, v):\n        \"\"\"Checks if the character entered is valid, by checking against the\n        phonetic vector. At least 1 of the 38 properties have to be\n        satisfied for a valid vector.\n        \"\"\"\n        return np.sum(v) > 0\n\n    def is_vowel(self, v):\n        \"\"\"Checks the property of the character (of being a vowel) selected against its phonetic vector\n            \"\"\"\n        return v[PVIDX_BT_VOWEL] == 1\n\n    def is_anusvaar(self, v):\n        \"\"\"Checks the property of the character (of having an anusvaar)\n        selected against its phonetic vector.\n        \"\"\"\n        return v[PVIDX_BT_ANUSVAAR] == 1\n\n    def is_plosive(self, v):\n        \"\"\"Checks the property of the character (of being a plosive\n        character) selected against its phonetic vector.\n         \"\"\"\n        return self.is_consonant(v) and self.get_property_vector(v, 'consonant_type')[0] == 1\n\n    def is_nukta(self,v):\n        \"\"\"Checks the property of the character (of having a nukta) selected\n        against its phonetic vector.\n        \"\"\"\n        return v[PVIDX_BT_NUKTA] == 1\n\n    def is_dependent_vowel(self, v):\n        \"\"\"Checks the property of the character (if it is a dependent\n        vowel) selected against its phonetic vector.\n        \"\"\"\n        return self.is_vowel(v) and v[PVIDX_VSTAT_DEP] == 1\n\n    def orthographic_syllabify(self, word):\n        \"\"\"Main syllablic function.\"\"\"\n        p_vectors = [self.get_phonetic_feature_vector(c, self.lang) for c in word]\n\n        syllables = []\n\n        for i in range(len(word)):\n            v = p_vectors[i]\n\n            syllables.append(word[i])\n\n            if i + 1 < len(word) and (not self.is_valid(p_vectors[i + 1]) or self.is_misc(p_vectors[i + 1])):\n                syllables.append(u' ')\n\n            elif not self.is_valid(v) or self.is_misc(v):\n                syllables.append(u' ')\n\n            elif self.is_vowel(v):\n\n                anu_nonplos = (i + 2 < len(word) and\n                               self.is_anusvaar(p_vectors[i + 1]) and\n                               not self.is_plosive(p_vectors[i + 2])\n                               )\n\n                anu_eow = (i + 2 == len(word) and\n                           self.is_anusvaar(p_vectors[i + 1]))\n\n                if not (anu_nonplos or anu_eow):\n                    syllables.append(u' ')\n\n            elif i + 1 < len(word) and (self.is_consonant(v) or self.is_nukta(v)):\n                if self.is_consonant(p_vectors[i + 1]):\n                    syllables.append(u' ')\n                elif self.is_vowel(p_vectors[i + 1]) and not self.is_dependent_vowel(p_vectors[i + 1]):\n                    syllables.append(u' ')\n                elif self.is_anusvaar(p_vectors[i + 1]):\n                    anu_nonplos = (i + 2 < len(word) and not self.is_plosive(p_vectors[i + 2]))\n\n                    anu_eow = i + 2 == len(word)\n\n                    if not (anu_nonplos or anu_eow):\n                        syllables.append(u' ')\n\n        return u''.join(syllables).strip().split(u' ')\n\n\nif __name__ == '__main__':\n    syllabifier = Syllabifier('hindi')\n    current = syllabifier.orthographic_syllabify('\u0928\u092e\u0938\u094d\u0924\u0947')\n    print(current)\n", "path": "cltk/stem/sanskrit/indian_syllabifier.py"}], "after_files": [{"content": "\"\"\"Every phonetic of every language is given similar positions in the vectors. Therefore transliterations\nhappen when each offset is calculated relative to the ranges of the languages specified.\nEvery phonetic has a dedicated phonetic vector which describes all the facets of the character, whether it is\na vowel or a consonant whe ther it has a halanta, etc.\n\nSource: https://github.com/anoopkunchukuttan/indic_nlp_library/blob/master/src/indicnlp/script/indic_scripts.py\n\"\"\"\n\nimport os\nimport csv\n\ntry:\n    import numpy as np\nexcept ImportError:\n    print('\"numpy\" is not installed.')\n    raise\n\n__author__ = ['Anoop Kunchukuttan']\n__license__ = 'GPLv3'\n\n\n# Indexes into the phonetic vector\nPVIDX_BT_VOWEL = 0\nPVIDX_BT_CONSONANT = 1\nPVIDX_BT_NUKTA = 2\nPVIDX_BT_HALANT = 3\nPVIDX_BT_ANUSVAAR = 4\nPVIDX_BT_MISC = 5\nPVIDX_BT_S = PVIDX_BT_VOWEL\nPVIDX_BT_E = PVIDX_BT_MISC + 1\n\nPVIDX_VSTAT_DEP = 12\n\nLC_TA = 'ta'\n\nLANGUAGE_NAME_TO_CODE = {'hindi': 'hi', 'sanskrit': 'sa', 'punjabi': 'pa', 'gujarati': 'gu', 'oriya': 'or',\n                         'tamil': 'ta', 'telegu': 'te', 'kannada': 'kn', 'malayalam': 'ml', 'sinhalese': 'si',\n                         'marathi': 'mr', 'konkan': 'kk', 'nepali': 'ne', 'sindhi': 'sd', 'bengali': 'bn',\n                         'assamese': 'as'}\n\n\n# The phonetics of every script exist in the ranges of the dictionary mentioned below\nSCRIPT_RANGES = {\n    'pa': [0x0a00, 0x0a7f],\n    'gu': [0x0a80, 0x0aff],\n    'or': [0x0b00, 0x0b7f],\n    'ta': [0x0b80, 0x0bff],\n    'te': [0x0c00, 0x0c7f],\n    'kn': [0x0c80, 0x0cff],\n    'ml': [0x0d00, 0x0d7f],\n    'si': [0x0d80, 0x0dff],\n    'hi': [0x0900, 0x097f],\n    'mr': [0x0900, 0x097f],\n    'kk': [0x0900, 0x097f],\n    'sa': [0x0900, 0x097f],\n    'ne': [0x0900, 0x097f],\n    'sd': [0x0900, 0x097f],\n    'bn': [0x0980, 0x09ff],\n    'as': [0x0980, 0x09ff],\n}\n\nCOORDINATED_RANGE_START_INCLUSIVE = 0\nCOORDINATED_RANGE_END_INCLUSIVE = 0x6f\n\nPV_PROP_RANGES = dict(basic_type=[0, 6], vowel_length=[6, 8], vowel_strength=[8, 11], vowel_status=[11, 13],\n                      consonant_type=[13, 18], articulation_place=[18, 23], aspiration=[23, 25], voicing=[25, 27],\n                      nasalization=[27, 29], vowel_horizontal=[29, 32], vowel_vertical=[32, 36],\n                      vowel_roundness=[36, 38])\n\nPHONETIC_VECTOR_START_OFFSET = 6\n\n\nclass Syllabifier:\n    \"\"\"Class for syllabalizing Indian language words.\"\"\"\n\n    def __init__(self, lang_name):\n        \"\"\"Setup values.\"\"\"\n\n        self.lang_name = lang_name\n        assert self.lang_name in LANGUAGE_NAME_TO_CODE.keys(), 'Language not available'\n        self.lang = LANGUAGE_NAME_TO_CODE[lang_name]\n\n        assert self.lang in SCRIPT_RANGES.keys()\n\n        self.all_phonetic_data, self.tamil_phonetic_data, self.all_phonetic_vectors, self.tamil_phonetic_vectors, self.phonetic_vector_length = self.get_lang_data()\n\n    def get_lang_data(self):\n        \"\"\"Define and call data for future use. Initializes and defines all\n        variables which define the phonetic vectors.\n        \"\"\"\n\n        root = os.path.expanduser('~')\n        csv_dir_path = os.path.join(root, 'cltk_data/sanskrit/model/sanskrit_models_cltk/phonetics')\n\n        all_phonetic_csv = os.path.join(csv_dir_path, 'all_script_phonetic_data.csv')\n        tamil_csv = os.path.join(csv_dir_path, 'tamil_script_phonetic_data.csv')\n\n        # Make helper function for this\n        with open(all_phonetic_csv,'r') as f:\n            reader = csv.reader(f, delimiter = ',', quotechar = '\"')\n            next(reader, None) # Skip headers\n            all_phonetic_data = [row for row in reader]\n\n        with open(tamil_csv,'r') as f:\n            reader = csv.reader(f, delimiter = ',', quotechar = '\"')\n            next(reader, None) # Skip headers\n            # tamil_phonetic_data = [row[PHONETIC_VECTOR_START_OFFSET:] for row in reader]\n            tamil_phonetic_data = [row for row in reader]\n\n        # Handle better?\n        all_phonetic_data = [[int(cell) if cell=='0' or cell=='1' else cell for cell in row] for row in all_phonetic_data]\n        tamil_phonetic_data = [[int(cell) if cell=='0' or cell=='1' else cell for cell in row] for row in tamil_phonetic_data]\n\n        all_phonetic_vectors = np.array([row[PHONETIC_VECTOR_START_OFFSET:] for row in all_phonetic_data])\n        tamil_phonetic_vectors = np.array([row[PHONETIC_VECTOR_START_OFFSET:] for row in tamil_phonetic_data])\n\n        phonetic_vector_length = all_phonetic_vectors.shape[1]\n\n        return all_phonetic_data, tamil_phonetic_data, all_phonetic_vectors, tamil_phonetic_vectors, phonetic_vector_length\n\n    @staticmethod\n    def in_coordinated_range_offset(c_offset):\n        \"\"\"Applicable to Brahmi derived Indic scripts. Used to determine\n        whether offset is of a  alphabetic character or not.\n        \"\"\"\n        return COORDINATED_RANGE_START_INCLUSIVE <= c_offset <= COORDINATED_RANGE_END_INCLUSIVE\n\n    def get_offset(self, c, lang):\n        \"\"\"Gets the offset; that is the relative position in the range of the\n        specified language.\n        \"\"\"\n        return ord(c) - SCRIPT_RANGES[lang][0]\n\n    def invalid_vector(self):\n        \"\"\"Returns an zero array of length 38\"\"\"\n        return np.array([0] * self.phonetic_vector_length)\n\n    def get_phonetic_info(self, lang):\n        \"\"\"For a specified language (lang), it returns the matrix and the vecto\n         containing specifications of the characters.\n         \"\"\"\n        phonetic_data = self.all_phonetic_data if lang != LC_TA else self.tamil_phonetic_data\n        phonetic_vectors = self.all_phonetic_vectors if lang != LC_TA else self.tamil_phonetic_vectors\n\n        return phonetic_data, phonetic_vectors\n\n    def get_phonetic_feature_vector(self, c, lang):\n        \"\"\"For a given character in a language, it gathers all the information related to it\n          (eg: whether fricative, plosive,etc)\"\"\"\n\n        offset = self.get_offset(c, lang)\n        if not self.in_coordinated_range_offset(offset):\n            return self.invalid_vector()\n\n        phonetic_data, phonetic_vectors = self.get_phonetic_info(lang)\n\n        # 'Valid Vector Representation' is the [5] column\n        if phonetic_data[offset][5] == 0:\n            return self.invalid_vector()\n\n        return phonetic_vectors[offset]\n\n    def get_property_vector(self, v, prop_name):\n        \"\"\"Returns the part of the vector corresponding to the required property\"\"\"\n        return v[PV_PROP_RANGES[prop_name][0]:PV_PROP_RANGES[prop_name][1]]\n\n\n    def is_consonant(self, v):\n        \"\"\"Checks the property of the character (of being a consonant)\n        selected against its phonetic vector.\n        \"\"\"\n        return v[PVIDX_BT_CONSONANT] == 1\n\n\n    def is_misc(self,v):\n        \"\"\"Checks the property of the character (of being miscellenous)\n        selected against its phonetic vector.\n        \"\"\"\n        return v[PVIDX_BT_MISC] == 1\n\n    def is_valid(self, v):\n        \"\"\"Checks if the character entered is valid, by checking against the\n        phonetic vector. At least 1 of the 38 properties have to be\n        satisfied for a valid vector.\n        \"\"\"\n        return np.sum(v) > 0\n\n    def is_vowel(self, v):\n        \"\"\"Checks the property of the character (of being a vowel) selected against its phonetic vector\n            \"\"\"\n        return v[PVIDX_BT_VOWEL] == 1\n\n    def is_anusvaar(self, v):\n        \"\"\"Checks the property of the character (of having an anusvaar)\n        selected against its phonetic vector.\n        \"\"\"\n        return v[PVIDX_BT_ANUSVAAR] == 1\n\n    def is_plosive(self, v):\n        \"\"\"Checks the property of the character (of being a plosive\n        character) selected against its phonetic vector.\n         \"\"\"\n        return self.is_consonant(v) and self.get_property_vector(v, 'consonant_type')[0] == 1\n\n    def is_nukta(self,v):\n        \"\"\"Checks the property of the character (of having a nukta) selected\n        against its phonetic vector.\n        \"\"\"\n        return v[PVIDX_BT_NUKTA] == 1\n\n    def is_dependent_vowel(self, v):\n        \"\"\"Checks the property of the character (if it is a dependent\n        vowel) selected against its phonetic vector.\n        \"\"\"\n        return self.is_vowel(v) and v[PVIDX_VSTAT_DEP] == 1\n\n    def orthographic_syllabify(self, word):\n        \"\"\"Main syllablic function.\"\"\"\n        p_vectors = [self.get_phonetic_feature_vector(c, self.lang) for c in word]\n\n        syllables = []\n\n        for i in range(len(word)):\n            v = p_vectors[i]\n\n            syllables.append(word[i])\n\n            if i + 1 < len(word) and (not self.is_valid(p_vectors[i + 1]) or self.is_misc(p_vectors[i + 1])):\n                syllables.append(u' ')\n\n            elif not self.is_valid(v) or self.is_misc(v):\n                syllables.append(u' ')\n\n            elif self.is_vowel(v):\n\n                anu_nonplos = (i + 2 < len(word) and\n                               self.is_anusvaar(p_vectors[i + 1]) and\n                               not self.is_plosive(p_vectors[i + 2])\n                               )\n\n                anu_eow = (i + 2 == len(word) and\n                           self.is_anusvaar(p_vectors[i + 1]))\n\n                if not (anu_nonplos or anu_eow):\n                    syllables.append(u' ')\n\n            elif i + 1 < len(word) and (self.is_consonant(v) or self.is_nukta(v)):\n                if self.is_consonant(p_vectors[i + 1]):\n                    syllables.append(u' ')\n                elif self.is_vowel(p_vectors[i + 1]) and not self.is_dependent_vowel(p_vectors[i + 1]):\n                    syllables.append(u' ')\n                elif self.is_anusvaar(p_vectors[i + 1]):\n                    anu_nonplos = (i + 2 < len(word) and not self.is_plosive(p_vectors[i + 2]))\n\n                    anu_eow = i + 2 == len(word)\n\n                    if not (anu_nonplos or anu_eow):\n                        syllables.append(u' ')\n\n        return u''.join(syllables).strip().split(u' ')\n\n\nif __name__ == '__main__':\n    syllabifier = Syllabifier('hindi')\n    current = syllabifier.orthographic_syllabify('\u0928\u092e\u0938\u094d\u0924\u0947')\n    print(current)\n", "path": "cltk/stem/sanskrit/indian_syllabifier.py"}]}