feat: add helm chart

k8s-deploy/databases/00-config.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 
 # Get the directory where this script is located
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
-source "$SCRIPT_DIR/scripts/common.sh"
+DATABASE_SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+source "$DATABASE_SCRIPT_DIR/scripts/common.sh"
 
 # Namespace configuration
 NAMESPACE="rag"
@@ -15,7 +15,7 @@ HELM_REPO="https://apecloud.github.io/helm-charts"
 # Set to true to enable the database, false to disable
 ENABLE_POSTGRESQL=true
 ENABLE_REDIS=false
-ENABLE_ELASTICSEARCH=false
 ENABLE_QDRANT=false
-ENABLE_MONGODB=false
 ENABLE_NEO4J=true
+ENABLE_ELASTICSEARCH=false
+ENABLE_MONGODB=false

k8s-deploy/databases/01-prepare.sh

@@ -1,14 +1,14 @@
 #!/bin/bash
 
 # Get the directory where this script is located
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+DATABASE_SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 # Load configuration file
-source "$SCRIPT_DIR/00-config.sh"
+source "$DATABASE_SCRIPT_DIR/00-config.sh"
 
 check_dependencies
 
 # Check if KubeBlocks is already installed, install it if it is not.
-source "$SCRIPT_DIR/install-kubeblocks.sh"
+source "$DATABASE_SCRIPT_DIR/install-kubeblocks.sh"
 
 # Create namespaces
 print "Creating namespaces..."

k8s-deploy/databases/02-install-database.sh

@@ -1,23 +1,61 @@
 #!/bin/bash
 
 # Get the directory where this script is located
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
-
-# KubeBlocks database installation script
-# Install all database clusters
+DATABASE_SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 
 # Load configuration file
-source "$SCRIPT_DIR/00-config.sh"
+source "$DATABASE_SCRIPT_DIR/00-config.sh"
 
 print "Installing database clusters..."
 
 # Install database clusters based on configuration
-[ "$ENABLE_POSTGRESQL" = true ] && print "Installing PostgreSQL cluster..." && helm upgrade --install pg-cluster kubeblocks/postgresql-cluster -f "$SCRIPT_DIR/postgresql/values.yaml" --namespace $NAMESPACE --version $ADDON_CLUSTER_CHART_VERSION
-[ "$ENABLE_REDIS" = true ] && print "Installing Redis cluster..." && helm upgrade --install redis-cluster kubeblocks/redis-cluster -f "$SCRIPT_DIR/redis/values.yaml" --namespace $NAMESPACE --version $ADDON_CLUSTER_CHART_VERSION
-[ "$ENABLE_ELASTICSEARCH" = true ] && print "Installing Elasticsearch cluster..." && helm upgrade --install es-cluster kubeblocks/elasticsearch-cluster -f "$SCRIPT_DIR/elasticsearch/values.yaml" --namespace $NAMESPACE --version $ADDON_CLUSTER_CHART_VERSION
-[ "$ENABLE_QDRANT" = true ] && print "Installing Qdrant cluster..." && helm upgrade --install qdrant-cluster kubeblocks/qdrant-cluster -f "$SCRIPT_DIR/qdrant/values.yaml" --namespace $NAMESPACE --version $ADDON_CLUSTER_CHART_VERSION
-[ "$ENABLE_MONGODB" = true ] && print "Installing MongoDB cluster..." && helm upgrade --install mongodb-cluster kubeblocks/mongodb-cluster -f "$SCRIPT_DIR/mongodb/values.yaml" --namespace $NAMESPACE --version $ADDON_CLUSTER_CHART_VERSION
-[ "$ENABLE_NEO4J" = true ] && print "Installing Neo4j cluster..." && helm upgrade --install neo4j-cluster kubeblocks/neo4j-cluster -f "$SCRIPT_DIR/neo4j/values.yaml" --namespace $NAMESPACE --version $ADDON_CLUSTER_CHART_VERSION
+[ "$ENABLE_POSTGRESQL" = true ] && print "Installing PostgreSQL cluster..." && helm upgrade --install pg-cluster kubeblocks/postgresql-cluster -f "$DATABASE_SCRIPT_DIR/postgresql/values.yaml" --namespace $NAMESPACE --version $ADDON_CLUSTER_CHART_VERSION
+[ "$ENABLE_REDIS" = true ] && print "Installing Redis cluster..." && helm upgrade --install redis-cluster kubeblocks/redis-cluster -f "$DATABASE_SCRIPT_DIR/redis/values.yaml" --namespace $NAMESPACE --version $ADDON_CLUSTER_CHART_VERSION
+[ "$ENABLE_ELASTICSEARCH" = true ] && print "Installing Elasticsearch cluster..." && helm upgrade --install es-cluster kubeblocks/elasticsearch-cluster -f "$DATABASE_SCRIPT_DIR/elasticsearch/values.yaml" --namespace $NAMESPACE --version $ADDON_CLUSTER_CHART_VERSION
+[ "$ENABLE_QDRANT" = true ] && print "Installing Qdrant cluster..." && helm upgrade --install qdrant-cluster kubeblocks/qdrant-cluster -f "$DATABASE_SCRIPT_DIR/qdrant/values.yaml" --namespace $NAMESPACE --version $ADDON_CLUSTER_CHART_VERSION
+[ "$ENABLE_MONGODB" = true ] && print "Installing MongoDB cluster..." && helm upgrade --install mongodb-cluster kubeblocks/mongodb-cluster -f "$DATABASE_SCRIPT_DIR/mongodb/values.yaml" --namespace $NAMESPACE --version $ADDON_CLUSTER_CHART_VERSION
+[ "$ENABLE_NEO4J" = true ] && print "Installing Neo4j cluster..." && helm upgrade --install neo4j-cluster kubeblocks/neo4j-cluster -f "$DATABASE_SCRIPT_DIR/neo4j/values.yaml" --namespace $NAMESPACE --version $ADDON_CLUSTER_CHART_VERSION
+
+# Wait for databases to be ready
+print "Waiting for databases to be ready..."
+TIMEOUT=600  # Set timeout to 10 minutes
+START_TIME=$(date +%s)
+
+while true; do
+  CURRENT_TIME=$(date +%s)
+  ELAPSED=$((CURRENT_TIME - START_TIME))
+
+  if [ $ELAPSED -gt $TIMEOUT ]; then
+    print_error "Timeout waiting for databases to be ready. Please check database status manually and try again"
+    exit 1
+  fi
+
+  # Build wait conditions for enabled databases
+  WAIT_CONDITIONS=()
+  [ "$ENABLE_POSTGRESQL" = true ] && WAIT_CONDITIONS+=("kubectl wait --for=condition=ready pods -l app.kubernetes.io/instance=pg-cluster -n $NAMESPACE --timeout=10s")
+  [ "$ENABLE_REDIS" = true ] && WAIT_CONDITIONS+=("kubectl wait --for=condition=ready pods -l app.kubernetes.io/instance=redis-cluster -n $NAMESPACE --timeout=10s")
+  [ "$ENABLE_ELASTICSEARCH" = true ] && WAIT_CONDITIONS+=("kubectl wait --for=condition=ready pods -l app.kubernetes.io/instance=es-cluster -n $NAMESPACE --timeout=10s")
+  [ "$ENABLE_QDRANT" = true ] && WAIT_CONDITIONS+=("kubectl wait --for=condition=ready pods -l app.kubernetes.io/instance=qdrant-cluster -n $NAMESPACE --timeout=10s")
+  [ "$ENABLE_MONGODB" = true ] && WAIT_CONDITIONS+=("kubectl wait --for=condition=ready pods -l app.kubernetes.io/instance=mongodb-cluster -n $NAMESPACE --timeout=10s")
+  [ "$ENABLE_NEO4J" = true ] && WAIT_CONDITIONS+=("kubectl wait --for=condition=ready pods -l app.kubernetes.io/instance=neo4j-cluster -n $NAMESPACE --timeout=10s")
+
+  # Check if all enabled databases are ready
+  ALL_READY=true
+  for CONDITION in "${WAIT_CONDITIONS[@]}"; do
+    if ! eval "$CONDITION &> /dev/null"; then
+      ALL_READY=false
+      break
+    fi
+  done
+
+  if [ "$ALL_READY" = true ]; then
+    print "All database pods are ready, continuing with deployment..."
+    break
+  fi
+
+  print "Waiting for database pods to be ready (${ELAPSED}s elapsed)..."
+  sleep 10
+done
 
 print_success "Database clusters installation completed!"
 print "Use the following command to check the status of installed clusters:"

k8s-deploy/databases/03-uninstall-database.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
 
 # Get the directory where this script is located
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+DATABASE_SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 
 # Load configuration file
-source "$SCRIPT_DIR/00-config.sh"
+source "$DATABASE_SCRIPT_DIR/00-config.sh"
 
 print "Uninstalling database clusters..."
 

k8s-deploy/databases/04-cleanup.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
 
 # Get the directory where this script is located
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+DATABASE_SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 
 # Load configuration file
-source "$SCRIPT_DIR/00-config.sh"
+source "$DATABASE_SCRIPT_DIR/00-config.sh"
 
 print "Uninstalling KubeBlocks database addons..."
 
@@ -18,7 +18,7 @@ print "Uninstalling KubeBlocks database addons..."
 
 print_success "Database addons uninstallation completed!"
 
-source "$SCRIPT_DIR/uninstall-kubeblocks.sh"
+source "$DATABASE_SCRIPT_DIR/uninstall-kubeblocks.sh"
 
 kubectl delete namespace $NAMESPACE
 kubectl delete namespace kb-system

k8s-deploy/databases/install-kubeblocks.sh

@@ -1,9 +1,9 @@
 #!/bin/bash
 
 # Get the directory where this script is located
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+DATABASE_SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 # Load configuration file
-source "$SCRIPT_DIR/00-config.sh"
+source "$DATABASE_SCRIPT_DIR/00-config.sh"
 
 # Check dependencies
 check_dependencies

k8s-deploy/databases/uninstall-kubeblocks.sh

@@ -1,9 +1,9 @@
 #!/bin/bash
 
 # Get the directory where this script is located
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+DATABASE_SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 # Load configuration file
-source "$SCRIPT_DIR/00-config.sh"
+source "$DATABASE_SCRIPT_DIR/00-config.sh"
 
 # Check dependencies
 print "Checking dependencies..."

k8s-deploy/install_lightrag.sh

@@ -2,27 +2,8 @@
 
 NAMESPACE=rag
 
-# Get the directory where this script is located
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 
-check_dependencies(){
-  echo "Checking dependencies..."
-  command -v kubectl >/dev/null 2>&1 || { echo "Error: kubectl command not found"; exit 1; }
-  command -v helm >/dev/null 2>&1 || { echo "Error: helm command not found"; exit 1; }
-
-  # Check if Kubernetes is available
-  echo "Checking if Kubernetes is available..."
-  kubectl cluster-info &>/dev/null
-  if [ $? -ne 0 ]; then
-      echo "Error: Kubernetes cluster is not accessible. Please ensure you have proper access to a Kubernetes cluster."
-      exit 1
-  fi
-  echo "Kubernetes cluster is accessible."
-}
-
-check_dependencies
-
-# Check and set environment variables
 if [ -z "$OPENAI_API_KEY" ]; then
   echo "OPENAI_API_KEY environment variable is not set"
   read -p "Enter your OpenAI API key: " OPENAI_API_KEY
@@ -39,59 +20,24 @@ if [ -z "$OPENAI_API_BASE" ]; then
   export OPENAI_API_BASE=$OPENAI_API_BASE
 fi
 
-# Check if databases are already installed, install them if not
-echo "Checking database installation status..."
-if ! kubectl get clusters -n rag pg-cluster &> /dev/null || ! kubectl get clusters -n rag neo4j-cluster &> /dev/null; then
-  echo "Databases not installed or incompletely installed, will install required databases first..."
-
-  # Install KubeBlocks (if not already installed)
-  echo "Preparing to install KubeBlocks and required components..."
-  bash "$SCRIPT_DIR/databases/01-prepare.sh"
-
-  # Install database clusters
-  echo "Installing database clusters..."
-  bash "$SCRIPT_DIR/databases/02-install-database.sh"
-
-  # Wait for databases to be ready
-  echo "Waiting for databases to be ready..."
-  TIMEOUT=300  # Set timeout to 5 minutes
-  START_TIME=$(date +%s)
-
-  while true; do
-    CURRENT_TIME=$(date +%s)
-    ELAPSED=$((CURRENT_TIME - START_TIME))
-
-    if [ $ELAPSED -gt $TIMEOUT ]; then
-      echo "Timeout waiting for databases to be ready. Please check database status manually and try again"
-      exit 1
-    fi
-
-    # Use kubectl wait to check if both databases are ready
-    if kubectl wait --for=condition=ready pods -l app.kubernetes.io/instance=pg-cluster -n rag --timeout=10s &> /dev/null &&
-       kubectl wait --for=condition=ready pods -l app.kubernetes.io/instance=neo4j-cluster -n rag --timeout=10s &> /dev/null; then
-      echo "Database pods are ready, continuing with LightRAG deployment..."
-      break
+# Install KubeBlocks (if not already installed)
+echo "Preparing to install KubeBlocks and required components..."
+bash "$SCRIPT_DIR/databases/01-prepare.sh"
+
+# Install database clusters
+echo "Installing database clusters..."
+bash "$SCRIPT_DIR/databases/02-install-database.sh"
+
+# Create vector extension in PostgreSQL if enabled
+if [ "$ENABLE_POSTGRESQL" = true ]; then
+    print "Waiting for PostgreSQL pods to be ready..."
+    if kubectl wait --for=condition=ready pods -l app.kubernetes.io/instance=pg-cluster -n $NAMESPACE --timeout=300s; then
+        print "Creating vector extension in PostgreSQL..."
+        kubectl exec -it $(kubectl get pods -l kubeblocks.io/role=primary,app.kubernetes.io/instance=pg-cluster -n $NAMESPACE -o name) -n $NAMESPACE -- psql -c "CREATE EXTENSION vector;"
+        print_success "Vector extension created successfully."
+    else
+        print "Warning: PostgreSQL pods not ready within timeout. Vector extension not created."
     fi
-
-    echo "Waiting for database pods to be ready..."
-    sleep 10
-  done
-else
-  echo "Databases already installed, checking if database pods are ready..."
-
-  # Verify that pods are ready before proceeding
-  echo "Waiting for database pods to be ready..."
-  if ! kubectl wait --for=condition=ready pods -l app.kubernetes.io/instance=pg-cluster -n rag --timeout=60s; then
-    echo "PostgreSQL pods are not ready. Please check database status manually."
-    exit 1
-  fi
-
-  if ! kubectl wait --for=condition=ready pods -l app.kubernetes.io/instance=neo4j-cluster -n rag --timeout=60s; then
-    echo "Neo4j pods are not ready. Please check database status manually."
-    exit 1
-  fi
-
-  echo "Database pods are ready, proceeding with LightRAG deployment..."
 fi
 
 # Get database passwords from Kubernetes secrets
@@ -110,6 +56,13 @@ if [ -z "$NEO4J_PASSWORD" ]; then
 fi
 export NEO4J_PASSWORD=$NEO4J_PASSWORD
 
+#REDIS_PASSWORD=$(kubectl get secrets -n rag redis-cluster-redis-account-default -o jsonpath='{.data.password}' | base64 -d)
+#if [ -z "$REDIS_PASSWORD" ]; then
+#  echo "Error: Could not retrieve Redis password. Make sure Redis is deployed and the secret exists."
+#  exit 1
+#fi
+#export REDIS_PASSWORD=$REDIS_PASSWORD
+
 echo "Deploying production LightRAG (using external databases)..."
 
 if ! kubectl get namespace rag &> /dev/null; then
@@ -129,7 +82,11 @@ helm upgrade --install lightrag $SCRIPT_DIR/lightrag \
   --set-string env.EMBEDDING_MODEL=text-embedding-ada-002 \
   --set-string env.EMBEDDING_DIM=1536 \
   --set-string env.EMBEDDING_BINDING_API_KEY=$OPENAI_API_KEY
+#  --set-string env.REDIS_URI="redis://default:${REDIS_PASSWORD}@redis-cluster-redis-redis:6379"
 
 # Wait for LightRAG pod to be ready
 echo "Waiting for LightRAG pod to be ready..."
 kubectl wait --for=condition=ready pod -l app.kubernetes.io/instance=lightrag --timeout=60s -n rag
+
+#echo "Current LightRAG Config: "
+#kubectl get secrets lightrag-env -o jsonpath='{.data.\.env}' -n rag | base64 -d

k8s-deploy/install_lightrag_dev.sh

@@ -2,7 +2,6 @@
 
 NAMESPACE=rag
 
-# Get the directory where this script is located
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 
 check_dependencies(){

k8s-deploy/lightrag/templates/deployment.yaml

@@ -24,6 +24,15 @@ spec:
             - name: http
               containerPort: {{ .Values.env.PORT }}
               protocol: TCP
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 5
+            timeoutSeconds: 2
+            successThreshold: 1
+            failureThreshold: 3
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           volumeMounts:

k8s-deploy/lightrag/values.yaml

@@ -38,6 +38,8 @@ env:
   EMBEDDING_BINDING_API_KEY:
   LIGHTRAG_KV_STORAGE: PGKVStorage
   LIGHTRAG_VECTOR_STORAGE: PGVectorStorage
+#  LIGHTRAG_KV_STORAGE: RedisKVStorage
+#  LIGHTRAG_VECTOR_STORAGE: QdrantVectorDBStorage
   LIGHTRAG_GRAPH_STORAGE: Neo4JStorage
   LIGHTRAG_DOC_STATUS_STORAGE: PGDocStatusStorage
   # Replace with your POSTGRES credentials
@@ -51,3 +53,6 @@ env:
   NEO4J_URI: neo4j://neo4j-cluster-neo4j:7687
   NEO4J_USERNAME: neo4j
   NEO4J_PASSWORD:
+  # Replace with your Qdrant credentials
+  QDRANT_URL: http://qdrant-cluster-qdrant-qdrant:6333
+  # REDIS_URI: redis://default:${REDIS_PASSWORD}@redis-cluster-redis-redis:6379