Added Kiswahili training script and training data

.gitattributes

@@ -1,35 +1,5 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Git attributes
+*.py text
+*.txt text
+*.md text
+*.json text

train_model.py

@@ -0,0 +1,126 @@
+import torch
+from transformers import (
+    AutoModelForCausalLM, 
+    AutoTokenizer, 
+    TrainingArguments, 
+    Trainer,
+    DataCollatorForLanguageModeling
+)
+from datasets import Dataset
+import os
+import glob
+
+class KiswahiliTrainer:
+    def __init__(self, model_name="distilgpt2"):
+        self.model_name = model_name
+        self.tokenizer = AutoTokenizer.from_pretrained(model_name)
+        self.tokenizer.pad_token = self.tokenizer.eos_token
+        
+        self.model = AutoModelForCausalLM.from_pretrained(model_name)
+        print("✅ Model and tokenizer loaded successfully!")
+    
+    def load_training_data(self, data_dir="./training_data"):
+        """Load all training data from text files"""
+        conversations = []
+        
+        # Get all text files in the data directory
+        text_files = glob.glob(os.path.join(data_dir, "*.txt"))
+        
+        for file_path in text_files:
+            print(f"📖 Loading data from: {file_path}")
+            try:
+                with open(file_path, 'r', encoding='utf-8') as f:
+                    content = f.read().strip()
+                    
+                    # Split by empty lines to get individual Q&A pairs
+                    pairs = [p.strip() for p in content.split('\n\n') if p.strip()]
+                    
+                    for pair in pairs:
+                        if 'User:' in pair and 'Bot:' in pair:
+                            conversations.append(pair)
+                            
+            except Exception as e:
+                print(f"❌ Error reading {file_path}: {e}")
+        
+        print(f"📊 Loaded {len(conversations)} training examples")
+        return conversations
+    
+    def prepare_dataset(self, conversations):
+        """Prepare the dataset for training"""
+        formatted_data = []
+        
+        for conv in conversations:
+            # Clean and format the conversation
+            formatted_conv = conv.replace('Bot:', 'Assistant:')
+            formatted_data.append({"text": formatted_conv})
+        
+        # Create dataset
+        dataset = Dataset.from_list(formatted_data)
+        
+        # Tokenize the dataset
+        def tokenize_function(examples):
+            return self.tokenizer(
+                examples["text"],
+                truncation=True,
+                max_length=512,
+                padding="max_length"
+            )
+        
+        tokenized_dataset = dataset.map(tokenize_function, batched=True)
+        return tokenized_dataset
+    
+    def train(self, output_dir="./trained_model"):
+        """Train the model"""
+        # Load training data
+        conversations = self.load_training_data()
+        
+        if not conversations:
+            print("❌ No training data found!")
+            return
+        
+        # Prepare dataset
+        dataset = self.prepare_dataset(conversations)
+        
+        # Set up training arguments
+        training_args = TrainingArguments(
+            output_dir=output_dir,
+            num_train_epochs=3,
+            per_device_train_batch_size=4,
+            save_steps=500,
+            save_total_limit=2,
+            prediction_loss_only=True,
+            logging_dir='./logs',
+            logging_steps=100,
+            learning_rate=5e-5,
+            warmup_steps=100,
+        )
+        
+        # Data collator
+        data_collator = DataCollatorForLanguageModeling(
+            tokenizer=self.tokenizer,
+            mlm=False  # We're doing causal LM, not masked LM
+        )
+        
+        # Initialize Trainer
+        trainer = Trainer(
+            model=self.model,
+            args=training_args,
+            train_dataset=dataset,
+            data_collator=data_collator,
+            tokenizer=self.tokenizer,
+        )
+        
+        # Start training
+        print("🚀 Starting training...")
+        trainer.train()
+        
+        # Save the model
+        print("💾 Saving model...")
+        trainer.save_model()
+        self.tokenizer.save_pretrained(output_dir)
+        print(f"✅ Training complete! Model saved to {output_dir}")
+
+# Run training
+if __name__ == "__main__":
+    trainer = KiswahiliTrainer()
+    trainer.train()

training_data/grammar.txt

@@ -0,0 +1,61 @@
+User: Unatumia "ni" na "si" vipi?
+Bot: "Ni" hutumika kwa uhalisiano mzuri. Mfano: "Mimi ni mwalimu". "Si" hutumika kukana. Mfano: "Mimi si mwalimu".
+
+User: Tofautisha kati ya "yeye" na "wewe"
+Bot: "Wewe" ni kiwakilishi cha mtu wa pili (you). "Yeye" ni kiwakilishi cha mtu wa tatu (he/she).
+
+User: Nini maana ya viwakilishi nafsi?
+Bot: Viwakilishi nafsi ni maneno yanayowakilisha watu kwenye sentensi. Kuna: mimi, wewe, yeye, sisi, nyinyi, wao.
+
+User: Je, "mtoto" ni ngeli gani?
+Bot: "Mtoto" ni ngeli ya KI-VI kwa umoja na WA-WA kwa wingi: mtoto/watoto.
+
+User: Toa mifano ya vitenzi vya kutendwa na kutendesha
+Bot: Kutendwa: "Kitabu kimesomwa na Juma". Kutendesha: "Juma amekisoma kitabu".
+
+User: Nini tofauti kati ya "liko" na "iko"?
+Bot: "Liko" hutumika kwa vitu vya ngeli ya LI-YA. "Iko" hutumika kwa vitu vya ngeli ya I-ZI na N-ZI.
+
+User: Vitenzi vya kupishana hutumikaje?
+Bot: Vitenzi vya kupishana huonyesha vitendo vinavyofanyika kwa pande zote mbili. Mfano: "Juma na Asha wanapendana".
+
+User: Nini maana ya kauli ya kutendwa?
+Bot: Kauli ya kutendwa huonyesha kitendo kinachotendwa kwa mtu au kitu. Mfano: "Chakula kimepakuliwa na mama".
+
+User: Taja vitenzi vya nyakati tofauti
+Bot: 1. Wakati uliopita: "nilisoma"
+2. Wakati ujao: "nitasoma" 
+3. Wakati uliopo: "ninasoma"
+
+User: Nini tofauti kati ya "huwa" na "awa"?
+Bot: "Huwa" ni kiwakilishi cha mtu wa tatu wingi (they). "Awa" si neno sahihi cha Kiswahili.
+
+User: Je, "kula" na "mla" ni vitenzi sawa?
+Bot: Hapana, "kula" ni kitenzi cha msingi (to eat). "Mla" ni neno linalomaanisha "mlaji" (eater) au hutumika kwa ngeli ya M-WA.
+
+User: Nini maana ya "na" katika sentensi?
+Bot: "Na" inaweza kuwa kiunganishi (and) au nyakati ya kisasa (present continuous). Mfano: "Juma na Asha" (and) au "Anasoma" (is reading).
+
+User: Vitenzi vya kutenda na kutendwa
+Bot: Kutenda: "Mwalimu anafundisha wanafunzi". Kutendwa: "Wanafunzi wanafundishwa na mwalimu".
+
+User: Nini maana ya kiulizi?
+Bot: Kiulizi ni neno linalotumika kuuliza swali. Kama: nani, nini, lini, wapi, vipi, kwa nini, gani.
+
+User: Tofautisha kati ya "hii" na "hili"
+Bot: "Hii" hutumika kwa vitu vya ngeli ya I-ZI. "Hili" hutumika kwa vitu vya ngeli ya LI-YA.
+
+User: Mfano wa sentensi yenye kihusishi
+Bot: Kihusishi ni neno kama "ya", "za", "la". Mfano: "Kitabu cha Juma", "Nyumba za watu".
+
+User: Nini maana ya kiwakilishi cha kutaja tena?
+Bot: Ni kiwakilishi kinachotumika badala ya jina lililotajwa tayari. Mfano: "Juma amekuja. Yeye ni mwalimu".
+
+User: Taja aina za maneno
+Bot: Kuna: nomino (nouns), vitenzi (verbs), vivumishi (adjectives), viwakilishi (pronouns), vihisishi (adverbs), viunganishi (conjunctions), vihisishi (prepositions), vikundi (interjections).
+
+User: Nini tofauti kati ya "kwanza" na "kwanza"?
+Bot: "Kwanza" inamaanisha "first". "Kwanza" si neno sahihi la Kiswahili; sahihi ni "kuanza" (to begin).
+
+User: Mfano wa sentensi yenye kirejeshi
+Bot: Kirejeshi ni neno kama "ambaye", "ambacho", "ambao". Mfano: "Mtoto ambaye anasoma", "Kitabu ambacho nasoma".

training_data/greetings.txt

@@ -0,0 +1,119 @@
+User: Habari yako?
+Bot: Nzuri sana, na wewe je?
+
+User: Hujambo?
+Bot: Sijambo!
+
+User: U hali gani?
+Bot: Niko salama, na wewe je?
+
+User: Mambo?
+Bot: Poa!
+
+User: Vipi?
+Bot: Safi!
+
+User: Shikamoo?
+Bot: Marahaba!
+
+User: Hamjambo?
+Bot: Hatujambo!
+
+User: Umeamkaje?
+Bot: Nimeamka salama, na wewe umeamkaje?
+
+User: Umeshindaje?
+Bot: Nimeshinda vizuri, na wewe umeshindaje?
+
+User: Hali za leo?
+Bot: Njema sana, na wewe hali zako?
+
+User: Habari za asubuhi?
+Bot: Nzuri sana, habari za asubuhi!
+
+User: Habari za mchana?
+Bot: Njema, habari za mchana!
+
+User: Habari za jioni?
+Bot: Zipo vizuri, habari za jioni!
+
+User: Habari za kazi?
+Bot: Kazi iendelee, Mungu akipenda!
+
+User: Habari za nyumbani?
+Bot: Zote vizuri, Mungu akipenda.
+
+User: Habari za familia?
+Bot: Wote wazima, na wa kwenu?
+
+User: Habari za safari?
+Bot: Safari salama, Mungu akipenda.
+
+User: Umetoka wapi?
+Bot: Nimetoka nyumbani/kazini/sokoni.
+
+User: Unaenda wapi?
+Bot: Naenda nyumbani/kazini/sokoni.
+
+User: Uko poa?
+Bot: Poa kama ndizi!
+
+User: Mzima?
+Bot: Mzima wa afya!
+
+User: Maisha yanaendelea vipi?
+Bot: Yanadunda, Mungu akipenda.
+
+User: Mungu awabariki
+Bot: Amen, na akubariki wewe pia.
+
+User: Umepotea wapi?
+Bot: Samahani, nilikuwa na shughuli nyingi.
+
+User: Karibu
+Bot: Asante sana kwa kunikaribisha.
+
+User: Asante
+Bot: Karibu sana.
+
+User: Asante sana
+Bot: Karibu tena.
+
+User: Pole na masaa
+Bot: Hamna shida, hakuna haraka.
+
+User: Pole na kazi
+Bot: Asante, kazi ni maisha.
+
+User: Hongera
+Bot: Asante sana, Mungu akipendelea.
+
+User: Usiku mwema
+Bot: Usiku mwema, lala salama.
+
+User: Kwaheri
+Bot: Kwaheri, tutaonana tena.
+
+User: Tutaonana
+Bot: Ndio, tutaonana baadaye.
+
+User: Baadaye
+Bot: Sawa, baadaye.
+
+User: Kesho
+Bot: Sawa, kesho.
+
+User: Haya
+Bot: Sawa, tuendelee.
+
+User: Sawa
+Bot: Sawa kabisa.
+
+User: Ahsante kwa msaada
+Bot: Karibu sana, ni furaha yangu.
+
+User: Samahani
+Bot: Hamna shida, pole sana.
+
+User: Pole
+Bot: Asante, Mungu atasaidia.

training_data/vocubulary.txt

@@ -0,0 +1,92 @@
+User: Nini maana ya "haraka"?
+Bot: "Haraka" inamaanisha kufanya kitu kwa spidi au bila kuchelewesha. Kwa Kiingereza: "quickly" or "fast".
+
+User: Nini maana ya "polepole"?
+Bot: "Polepole" inamaanisha kufanya kitu kwa utulivu au kwa mwendo wa chini. Kwa Kiingereza: "slowly".
+
+User: Tafsiri "house" kwa Kiswahili
+Bot: "House" inatafsiriwa kama "nyumba" kwa Kiswahili.
+
+User: Nini maana ya "chakula"?
+Bot: "Chakula" ni kitu tunachokula kwa kulishiba au kupata nishati. Kwa Kiingereza: "food".
+
+User: Tofautisha kati ya "mkate" na "chapo"
+Bot: "Mkate" ni vyakula vilivyookwa kwa unga wa ngano. "Chapo" ni aina ya mkate ulio na mafuta na umekatwa kwa vipande.
+
+User: Nini maana ya "maji"?
+Bot: "Maji" ni kiowevu kisicho na rangi au harufu kinachotiririka. Kwa Kiingereza: "water".
+
+User: Tafsiri "school" kwa Kiswahili
+Bot: "School" inatafsiriwa kama "shule" kwa Kiswahili.
+
+User: Nini maana ya "rafiki"?
+Bot: "Rafiki" ni mtu unaoyefahamiana naye vizuri na una uhusiano mwema. Kwa Kiingereza: "friend".
+
+User: Taja maneno matatu yanayohusiana na "mlimajengu"
+Bot: 1. Mlima - mountain
+2. Kilele - peak
+3. Mtelemko - slope
+
+User: Nini maana ya "soko"?
+Bot: "Soko" ni mahala panapouzwa na kununuliwa bidhaa mbalimbali. Kwa Kiingereza: "market".
+
+User: Tafsiri "book" kwa Kiswahili
+Bot: "Book" inatafsiriwa kama "kitabu" kwa Kiswahili.
+
+User: Nini maana ya "mwalimu"?
+Bot: "Mwalimu" ni mtu anayefundisha watu wengine. Kwa Kiingereza: "teacher".
+
+User: Tofautisha kati ya "meza" na "kiti"
+Bot: "Meza" ni fanicha ya gorofa inayotumika kuwekea vitu. "Kiti" ni fanicha ya kukalia.
+
+User: Nini maana ya "moto"?
+Bot: "Moto" inaweza kumaanisha "fire" (moto) au "child" (mtoto mdogo) kulingana na muktadha.
+
+User: Tafsiri "family" kwa Kiswahili
+Bot: "Family" inatafsiriwa kama "familia" au "jamii" kwa Kiswahili.
+
+User: Nini maana ya "kupika"?
+Bot: "Kupika" ni kitendo cha kuandaa chakula kwa kutumia joto. Kwa Kiingereza: "to cook".
+
+User: Taja maneno matatu ya rangi
+Bot: 1. Nyekundu - red
+2. Blu - blue
+3. Kijani - green
+
+User: Nini maana ya "mji"?
+Bot: "Mji" ni eneo kubwa lenye watu wengi na majengo. Kwa Kiingereza: "city" or "town".
+
+User: Tafsiri "animal" kwa Kiswahili
+Bot: "Animal" inatafsiriwa kama "mnyama" kwa Kiswahili.
+
+User: Nini maana ya "mvua"?
+Bot: "Mvua" ni maji yanayomwagika kutoka angani kama matone. Kwa Kiingereza: "rain".
+
+User: Tofautisha kati ya "mto" na "ziwa"
+Bot: "Mto" ni mkondo wa maji unaotiririka. "Ziwa" ni mkusanyiko wa maji uliowekwa kwenye bonde.
+
+User: Nini maana ya "jua"?
+Bot: "Jua" ni nyota inayotoa mwanga na joto duniani. Kwa Kiingereza: "sun".
+
+User: Tafsiri "clothes" kwa Kiswahili
+Bot: "Clothes" inatafsiriwa kama "nguo" kwa Kiswahili.
+
+User: Nini maana ya "kulala"?
+Bot: "Kulala" ni kitendo cha kupumzika kwa macho yamefungwa. Kwa Kiingereza: "to sleep".
+
+User: Taja maneno matatu ya nambari
+Bot: 1. Moja - one
+2. Mbili - two
+3. Tatu - three
+
+User: Nini maana ya "barabara"?
+Bot: "Barabara" ni njia pana iliyotengenezwa kwa magari kupita. Kwa Kiingereza: "road".
+
+User: Tafsiri "money" kwa Kiswahili
+Bot: "Money" inatafsiriwa kama "pesa" kwa Kiswahili.
+
+User: Nini maana ya "kusoma"?
+Bot: "Kusoma" inaweza kumaanisha kusoma vitabu au kusoma shule. Kwa Kiingereza: "to read" or "to study".
+
+User: Tofautisha kati ya "mwezi" na "mwanga"
+Bot: "Mwezi" ni mwangaza unaotoka kwenye mwezi usiku. "Mwanga" ni uangaza wowote unaotoka kwenye vyanzo mbalimbali.