diff --git a/.gitattributes b/.gitattributes index 1abc6d15eb80582bc2bc48e8b790dd790b00d25b..a6344aac8c09253b3b630fb776ae94478aa0275b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,35 @@ -Animated_Logo_Video_Ready.gif filter=lfs diff=lfs merge=lfs -text +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore index 3f72906b798b3435d6ec00d45b2d8c4a002be398..0f25b866ecf0fddab1b5836b418c1966b98cb00f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +.gradio/ + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -19,18 +21,16 @@ lib64/ parts/ sdist/ var/ +wheels/ +share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST -# Virtual environments -venv/ -env/ -ENV/ -.venv/ - # PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec @@ -48,34 +48,115 @@ htmlcov/ nosetests.xml coverage.xml *.cover +*.py,cover .hypothesis/ .pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ # Jupyter Notebook .ipynb_checkpoints +# IPython +profile_default/ +ipython_config.py + # pyenv -.python-version +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site # mypy .mypy_cache/ .dmypy.json +dmypy.json # Pyre type checker .pyre/ -# Gradio cache -log/ -logs/ - -# System files -.DS_Store -Thumbs.db +# pytype static type analyzer +.pytype/ -# Lock files -uv.lock -poetry.lock -Pipfile.lock +# Cython debug symbols +cython_debug/ -# VSCode -.vscode/ \ No newline at end of file +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4bc0c623a659d38b912b9efd8443b18bc4922d49 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,32 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-executables-have-shebangs + - id: check-json + - id: check-merge-conflict + - id: check-shebang-scripts-are-executable + - id: check-toml + - id: check-yaml + - id: end-of-file-fixer + - id: mixed-line-ending + args: ["--fix=lf"] + - id: requirements-txt-fixer + - id: trailing-whitespace + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.8.6 + hooks: + - id: ruff + args: ["--fix"] + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.14.1 + hooks: + - id: mypy + args: ["--ignore-missing-imports"] + additional_dependencies: + [ + "types-python-slugify", + "types-requests", + "types-PyYAML", + "types-pytz", + ] diff --git a/.python-version b/.python-version new file mode 100644 index 0000000000000000000000000000000000000000..c8cfe3959183f8e9a50f83f54cd723f2dc9c252d --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.10 diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000000000000000000000000000000000000..854d34838a8695341a86e1cfa239be0014098058 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,8 @@ +{ + "recommendations": [ + "ms-python.python", + "charliermarsh.ruff", + "streetsidesoftware.code-spell-checker", + "tamasfe.even-better-toml" + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000000000000000000000000000000000..6b1ad68d2449998e1085249c2f4828e886ff47e5 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,16 @@ +{ + "editor.formatOnSave": true, + "files.insertFinalNewline": false, + "[python]": { + "editor.defaultFormatter": "charliermarsh.ruff", + "editor.formatOnType": true, + "editor.codeActionsOnSave": { + "source.fixAll.ruff": "explicit" + } + }, + "[jupyter]": { + "files.insertFinalNewline": false + }, + "notebook.output.scrolling": true, + "notebook.formatOnSave.enabled": true +} diff --git a/README.md b/README.md index 02f4fc48ab74bbdfdd926ac7d24fb822a106adb4..282f3700195e90b2e5a2446f8d96915eeb7cd3b9 100644 --- a/README.md +++ b/README.md @@ -4,132 +4,10 @@ emoji: 🏒 colorFrom: indigo colorTo: indigo sdk: gradio -sdk_version: 5.38.2 +sdk_version: 5.23.3 app_file: app.py pinned: false disable_embedding: true -hf_oauth: true -hf_oauth_scopes: -- manage-repos --- -# AnyCoder - AI Code Generator - -AnyCoder is an AI-powered code generator that helps you create applications by describing them in plain English. It supports multiple AI models, multimodal input, website redesign, and one-click deployment to Hugging Face Spaces. The UI is built with Gradio theming for a minimal, modern experience. - -## Features - -- **Multi-Model Support**: Choose from Moonshot Kimi-K2, DeepSeek V3, DeepSeek R1, ERNIE-4.5-VL, MiniMax M1, Qwen3-235B-A22B, Qwen3-30B-A3B-Instruct-2507, Qwen3-30B-A3B-Thinking-2507, SmolLM3-3B, and GLM-4.1V-9B-Thinking -- **Flexible Input**: Describe your app in text, upload a UI design image (for multimodal models), provide a reference file (PDF, TXT, MD, CSV, DOCX, or image), or enter a website URL for redesign -- **Web Search Integration**: Enable real-time web search (Tavily, with advanced search depth) to enhance code generation with up-to-date information and best practices -- **Code Generation**: Generate code in HTML, Python, JS, and more. Special support for transformers.js apps (outputs index.html, index.js, style.css) -- **Live Preview**: Instantly preview generated HTML in a sandboxed iframe -- **Modify Existing Code**: Use search/replace block format to update generated HTML -- **One-Click Deployment**: Deploy your app to Hugging Face Spaces (Gradio, Streamlit, Static HTML, or Transformers.js) with OAuth login -- **History & Examples**: Chat-like history of all interactions and quick example prompts for fast prototyping -- **Minimal, Modern UI**: Built with Gradio 5.x, using only built-in theming and styling (no custom CSS) - -## Installation - -1. Clone the repository: -```bash -git clone -cd anycoder -``` -2. Install dependencies: -```bash -pip install -r requirements.txt -``` -3. Set up environment variables: -```bash -export HF_TOKEN="your_huggingface_token" -export TAVILY_API_KEY="your_tavily_api_key" # Optional, for web search feature -export DASHSCOPE_API_KEY="your_dashscope_api_key" # Required for Qwen3-30B models via DashScope -``` - -## Usage - -1. Run the application: -```bash -python app.py -``` -2. Open your browser and navigate to the provided URL -3. Describe your application in the text input field, or: - - Upload a UI design image (for ERNIE-4.5-VL or GLM-4.1V-9B-Thinking) - - Upload a reference file (PDF, TXT, MD, CSV, DOCX, or image) - - Enter a website URL for redesign (the app will extract and analyze the HTML and content) - - Enable web search for up-to-date information - - Choose a different AI model or code language -4. Click "Generate" to create your code -5. View the generated code in the Code tab or see it in action in the Preview tab -6. Use the History tab to review previous generations -7. **Deploy to Space**: Enter a title, select SDK, and click "πŸš€ Deploy App" to publish your application (OAuth login required) - -## Supported Models - -- Moonshot Kimi-K2 -- DeepSeek V3 -- DeepSeek R1 -- ERNIE-4.5-VL (multimodal) -- MiniMax M1 -- Qwen3-235B-A22B -- Qwen3-4B-Instruct-2507 -- Qwen3-4B-Thinking-2507 -- Qwen3-30B-A3B-Instruct-2507 (via DashScope) -- Qwen3-30B-A3B-Thinking-2507 (via DashScope) -- SmolLM3-3B -- GLM-4.1V-9B-Thinking (multimodal) - -## Input Options - -- **Text Prompt**: Describe your app or code requirements -- **Image Upload**: For multimodal models, upload a UI design image to generate code from visuals -- **File Upload**: Provide a reference file (PDF, TXT, MD, CSV, DOCX, or image) for code generation or text extraction (OCR for images) -- **Website URL**: Enter a URL to extract and redesign the website (HTML and content are analyzed and modernized) - -## Web Search Feature - -- Enable the "Web search" toggle to use Tavily for real-time information (requires TAVILY_API_KEY) -- Uses advanced search depth for best results - -## Code Generation & Modification - -- Generates code in HTML, Python, JS, and more (selectable via dropdown) -- Special support for transformers.js apps (outputs index.html, index.js, style.css) -- Svelte apps -- For HTML, provides a live preview in a sandboxed iframe -- For modification requests, uses a search/replace block format to update existing HTML - -## Deployment - -- Deploy generated apps to Hugging Face Spaces directly from the UI -- Supported SDKs: Gradio (Python), Streamlit (Python), Static (HTML), Transformers.js -- OAuth login with Hugging Face is required for deployment to user-owned Spaces - -## History & Examples - -- Maintains a chat-like history of user/assistant interactions -- Quick example prompts are available in the sidebar for fast prototyping - -## UI/UX - -- Built with Gradio 5.x, using only Gradio's built-in theming and styling (no custom CSS) -- Minimal, uncluttered sidebar and interface - -## Environment Variables - -- `HF_TOKEN`: Your Hugging Face API token (required) -- `TAVILY_API_KEY`: Your Tavily API key (optional, for web search) - -## Project Structure - -``` -anycoder/ -β”œβ”€β”€ app.py # Main application (all logic and UI) -β”œβ”€β”€ requirements.txt -β”œβ”€β”€ README.md # This file -``` - -## License - -[Add your license information here] \ No newline at end of file +Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference \ No newline at end of file diff --git a/app.py b/app.py index 2cc964951de0e4e2ff4c0b5c38865c5fff007a4e..7017c18c09b63735c96780a229fed2c8da9329c9 100644 --- a/app.py +++ b/app.py @@ -1,4339 +1,54 @@ -import os -import re -from http import HTTPStatus -from typing import Dict, List, Optional, Tuple -import base64 -import mimetypes -import PyPDF2 -import docx -import cv2 -import numpy as np -from PIL import Image -import pytesseract -import requests -from urllib.parse import urlparse, urljoin -from bs4 import BeautifulSoup -import html2text -import json -import time -import webbrowser -import urllib.parse -import copy -import html - +from app_huggingface import demo as demo_huggingface +from app_gemini_coder import demo as demo_gemini +from utils import get_app import gradio as gr -from huggingface_hub import InferenceClient -from tavily import TavilyClient -from huggingface_hub import HfApi -import tempfile -from openai import OpenAI -from mistralai import Mistral - -# Gradio supported languages for syntax highlighting -GRADIO_SUPPORTED_LANGUAGES = [ - "python", "c", "cpp", "markdown", "latex", "json", "html", "css", "javascript", "jinja2", "typescript", "yaml", "dockerfile", "shell", "r", "sql", "sql-msSQL", "sql-mySQL", "sql-mariaDB", "sql-sqlite", "sql-cassandra", "sql-plSQL", "sql-hive", "sql-pgSQL", "sql-gql", "sql-gpSQL", "sql-sparkSQL", "sql-esper", None -] - -def get_gradio_language(language): - # Map composite options to a supported syntax highlighting - if language == "streamlit": - return "python" - if language == "gradio": - return "python" - return language if language in GRADIO_SUPPORTED_LANGUAGES else None - -# Search/Replace Constants -SEARCH_START = "<<<<<<< SEARCH" -DIVIDER = "=======" -REPLACE_END = ">>>>>>> REPLACE" - -# Configuration -HTML_SYSTEM_PROMPT = """ONLY USE HTML, CSS AND JAVASCRIPT. If you want to use ICON make sure to import the library first. Try to create the best UI possible by using only HTML, CSS and JAVASCRIPT. MAKE IT RESPONSIVE USING MODERN CSS. Use as much as you can modern CSS for the styling, if you can't do something with modern CSS, then use custom CSS. Also, try to elaborate as much as you can, to create something unique. ALWAYS GIVE THE RESPONSE INTO A SINGLE HTML FILE - -For website redesign tasks: -- Use the provided original HTML code as the starting point for redesign -- Preserve all original content, structure, and functionality -- Keep the same semantic HTML structure but enhance the styling -- Reuse all original images and their URLs from the HTML code -- Create a modern, responsive design with improved typography and spacing -- Use modern CSS frameworks and design patterns -- Ensure accessibility and mobile responsiveness -- Maintain the same navigation and user flow -- Enhance the visual design while keeping the original layout structure - -If an image is provided, analyze it and use the visual information to better understand the user's requirements. - -Always respond with code that can be executed or rendered directly. - -Always output only the HTML code inside a ```html ... ``` code block, and do not include any explanations or extra text. Do NOT add the language name at the top of the code output.""" - -TRANSFORMERS_JS_SYSTEM_PROMPT = """You are an expert web developer creating a transformers.js application. You will generate THREE separate files: index.html, index.js, and style.css. - -IMPORTANT: You MUST output ALL THREE files in the following format: - -```html - -``` - -```javascript -// index.js content here -``` - -```css -/* style.css content here */ -``` - -Requirements: -1. Create a modern, responsive web application using transformers.js -2. Use the transformers.js library for AI/ML functionality -3. Create a clean, professional UI with good user experience -4. Make the application fully responsive for mobile devices -5. Use modern CSS practices and JavaScript ES6+ features -6. Include proper error handling and loading states -7. Follow accessibility best practices - -The index.html should contain the basic HTML structure and link to the CSS and JS files. -The index.js should contain all the JavaScript logic including transformers.js integration. -The style.css should contain all the styling for the application. - -Always output only the three code blocks as shown above, and do not include any explanations or extra text.""" - -SVELTE_SYSTEM_PROMPT = """You are an expert Svelte developer creating a modern Svelte application. You will generate ONLY the custom files that need user-specific content for the user's requested application. - -IMPORTANT: You MUST output files in the following format. Generate ONLY the files needed for the user's specific request: - -```svelte - -``` - -```css -/* src/app.css content here */ -``` - -If you need additional components for the user's specific app, add them like: -```svelte - -``` - -Requirements: -1. Create a modern, responsive Svelte application based on the user's specific request -2. Use TypeScript for better type safety -3. Create a clean, professional UI with good user experience -4. Make the application fully responsive for mobile devices -5. Use modern CSS practices and Svelte best practices -6. Include proper error handling and loading states -7. Follow accessibility best practices -8. Use Svelte's reactive features effectively -9. Include proper component structure and organization -10. Generate ONLY components that are actually needed for the user's requested application - -Files you should generate: -- src/App.svelte: Main application component (ALWAYS required) -- src/app.css: Global styles (ALWAYS required) -- src/lib/[ComponentName].svelte: Additional components (ONLY if needed for the user's specific app) - -The other files (index.html, package.json, vite.config.ts, tsconfig files, svelte.config.js, src/main.ts, src/vite-env.d.ts) are provided by the Svelte template and don't need to be generated. - -Always output only the two code blocks as shown above, and do not include any explanations or extra text.""" - -SVELTE_SYSTEM_PROMPT_WITH_SEARCH = """You are an expert Svelte developer creating a modern Svelte application. You have access to real-time web search. When needed, use web search to find the latest information, best practices, or specific Svelte technologies. - -You will generate ONLY the custom files that need user-specific content. - -IMPORTANT: You MUST output ONLY the custom files in the following format: - -```svelte - -``` - -```css -/* src/app.css content here --> -``` - -Requirements: -1. Create a modern, responsive Svelte application -2. Use TypeScript for better type safety -3. Create a clean, professional UI with good user experience -4. Make the application fully responsive for mobile devices -5. Use modern CSS practices and Svelte best practices -6. Include proper error handling and loading states -7. Follow accessibility best practices -8. Use Svelte's reactive features effectively -9. Include proper component structure and organization -10. Use web search to find the latest Svelte patterns, libraries, and best practices - -The files you generate are: -- src/App.svelte: Main application component (your custom app logic) -- src/app.css: Global styles (your custom styling) - -The other files (index.html, package.json, vite.config.ts, tsconfig files, svelte.config.js, src/main.ts, src/vite-env.d.ts) are provided by the Svelte template and don't need to be generated. - -Always output only the two code blocks as shown above, and do not include any explanations or extra text.""" - -TRANSFORMERS_JS_SYSTEM_PROMPT_WITH_SEARCH = """You are an expert web developer creating a transformers.js application. You have access to real-time web search. When needed, use web search to find the latest information, best practices, or specific technologies for transformers.js. - -You will generate THREE separate files: index.html, index.js, and style.css. - -IMPORTANT: You MUST output ALL THREE files in the following format: - -```html - -``` - -```javascript -// index.js content here -``` - -```css -/* style.css content here */ -``` - -Requirements: -1. Create a modern, responsive web application using transformers.js -2. Use the transformers.js library for AI/ML functionality -3. Use web search to find current best practices and latest transformers.js features -4. Create a clean, professional UI with good user experience -5. Make the application fully responsive for mobile devices -6. Use modern CSS practices and JavaScript ES6+ features -7. Include proper error handling and loading states -8. Follow accessibility best practices - -The index.html should contain the basic HTML structure and link to the CSS and JS files. -The index.js should contain all the JavaScript logic including transformers.js integration. -The style.css should contain all the styling for the application. - -Always output only the three code blocks as shown above, and do not include any explanations or extra text.""" - -GENERIC_SYSTEM_PROMPT = """You are an expert {language} developer. Write clean, idiomatic, and runnable {language} code for the user's request. If possible, include comments and best practices. Output ONLY the code inside a ``` code block, and do not include any explanations or extra text. If the user provides a file or other context, use it as a reference. If the code is for a script or app, make it as self-contained as possible. Do NOT add the language name at the top of the code output.""" - -# System prompt with search capability -HTML_SYSTEM_PROMPT_WITH_SEARCH = """ONLY USE HTML, CSS AND JAVASCRIPT. If you want to use ICON make sure to import the library first. Try to create the best UI possible by using only HTML, CSS and JAVASCRIPT. MAKE IT RESPONSIVE USING MODERN CSS. Use as much as you can modern CSS for the styling, if you can't do something with modern CSS, then use custom CSS. Also, try to elaborate as much as you can, to create something unique. ALWAYS GIVE THE RESPONSE INTO A SINGLE HTML FILE - -You have access to real-time web search. When needed, use web search to find the latest information, best practices, or specific technologies. - -For website redesign tasks: -- Use the provided original HTML code as the starting point for redesign -- Preserve all original content, structure, and functionality -- Keep the same semantic HTML structure but enhance the styling -- Reuse all original images and their URLs from the HTML code -- Use web search to find current design trends and best practices for the specific type of website -- Create a modern, responsive design with improved typography and spacing -- Use modern CSS frameworks and design patterns -- Ensure accessibility and mobile responsiveness -- Maintain the same navigation and user flow -- Enhance the visual design while keeping the original layout structure - -If an image is provided, analyze it and use the visual information to better understand the user's requirements. - -Always respond with code that can be executed or rendered directly. - -Always output only the HTML code inside a ```html ... ``` code block, and do not include any explanations or extra text. Do NOT add the language name at the top of the code output.""" - -GENERIC_SYSTEM_PROMPT_WITH_SEARCH = """You are an expert {language} developer. You have access to real-time web search. When needed, use web search to find the latest information, best practices, or specific technologies for {language}. - -Write clean, idiomatic, and runnable {language} code for the user's request. If possible, include comments and best practices. Output ONLY the code inside a ``` code block, and do not include any explanations or extra text. If the user provides a file or other context, use it as a reference. If the code is for a script or app, make it as self-contained as possible. Do NOT add the language name at the top of the code output.""" - -# Follow-up system prompt for modifying existing HTML files -FollowUpSystemPrompt = f"""You are an expert web developer modifying an existing project. -The user wants to apply changes based on their request. -You MUST output ONLY the changes required using the following SEARCH/REPLACE block format. Do NOT output the entire file. -Explain the changes briefly *before* the blocks if necessary, but the code changes THEMSELVES MUST be within the blocks. - -IMPORTANT: When the user reports an ERROR MESSAGE, analyze it carefully to determine which file needs fixing: -- ImportError/ModuleNotFoundError β†’ Fix requirements.txt by adding missing packages -- Syntax errors in Python code β†’ Fix app.py or the main Python file -- HTML/CSS/JavaScript errors β†’ Fix the respective HTML/CSS/JS files -- Configuration errors β†’ Fix config files, Docker files, etc. - -For Python applications (Gradio/Streamlit), the project structure typically includes: -- app.py (main application file) -- requirements.txt (dependencies) -- Other supporting files as needed - -Format Rules: -1. Start with {SEARCH_START} -2. Provide the exact lines from the current code that need to be replaced. -3. Use {DIVIDER} to separate the search block from the replacement. -4. Provide the new lines that should replace the original lines. -5. End with {REPLACE_END} -6. You can use multiple SEARCH/REPLACE blocks if changes are needed in different parts of the file. -7. To insert code, use an empty SEARCH block (only {SEARCH_START} and {DIVIDER} on their lines) if inserting at the very beginning, otherwise provide the line *before* the insertion point in the SEARCH block and include that line plus the new lines in the REPLACE block. -8. To delete code, provide the lines to delete in the SEARCH block and leave the REPLACE block empty (only {DIVIDER} and {REPLACE_END} on their lines). -9. IMPORTANT: The SEARCH block must *exactly* match the current code, including indentation and whitespace. -10. For multi-file projects, specify which file you're modifying by starting with the filename before the search/replace block. - -Example Modifying Code: -``` -Some explanation... -{SEARCH_START} -

Old Title

-{DIVIDER} -

New Title

-{REPLACE_END} -{SEARCH_START} - -{DIVIDER} - - -{REPLACE_END} -``` - -Example Fixing Dependencies (requirements.txt): -``` -Adding missing dependency to fix ImportError... -=== requirements.txt === -{SEARCH_START} -gradio -streamlit -{DIVIDER} -gradio -streamlit -mistral-common -{REPLACE_END} -``` - -Example Deleting Code: -``` -Removing the paragraph... -{SEARCH_START} -

This paragraph will be deleted.

-{DIVIDER} -{REPLACE_END} -```""" - -# Follow-up system prompt for modifying existing transformers.js applications -TransformersJSFollowUpSystemPrompt = f"""You are an expert web developer modifying an existing transformers.js application. -The user wants to apply changes based on their request. -You MUST output ONLY the changes required using the following SEARCH/REPLACE block format. Do NOT output the entire file. -Explain the changes briefly *before* the blocks if necessary, but the code changes THEMSELVES MUST be within the blocks. - -IMPORTANT: When the user reports an ERROR MESSAGE, analyze it carefully to determine which file needs fixing: -- JavaScript errors/module loading issues β†’ Fix index.js -- HTML rendering/DOM issues β†’ Fix index.html -- Styling/visual issues β†’ Fix style.css -- CDN/library loading errors β†’ Fix script tags in index.html - -The transformers.js application consists of three files: index.html, index.js, and style.css. -When making changes, specify which file you're modifying by starting your search/replace blocks with the file name. - -Format Rules: -1. Start with {SEARCH_START} -2. Provide the exact lines from the current code that need to be replaced. -3. Use {DIVIDER} to separate the search block from the replacement. -4. Provide the new lines that should replace the original lines. -5. End with {REPLACE_END} -6. You can use multiple SEARCH/REPLACE blocks if changes are needed in different parts of the file. -7. To insert code, use an empty SEARCH block (only {SEARCH_START} and {DIVIDER} on their lines) if inserting at the very beginning, otherwise provide the line *before* the insertion point in the SEARCH block and include that line plus the new lines in the REPLACE block. -8. To delete code, provide the lines to delete in the SEARCH block and leave the REPLACE block empty (only {DIVIDER} and {REPLACE_END} on their lines). -9. IMPORTANT: The SEARCH block must *exactly* match the current code, including indentation and whitespace. - -Example Modifying HTML: -``` -Changing the title in index.html... -=== index.html === -{SEARCH_START} - Old Title -{DIVIDER} - New Title -{REPLACE_END} -``` - -Example Modifying JavaScript: -``` -Adding a new function to index.js... -=== index.js === -{SEARCH_START} -// Existing code -{DIVIDER} -// Existing code - -function newFunction() {{ - console.log("New function added"); -}} -{REPLACE_END} -``` - -Example Modifying CSS: -``` -Changing background color in style.css... -=== style.css === -{SEARCH_START} -body {{ - background-color: white; -}} -{DIVIDER} -body {{ - background-color: #f0f0f0; -}} -{REPLACE_END} -``` - -Example Fixing Library Loading Error: -``` -Fixing transformers.js CDN loading error... -=== index.html === -{SEARCH_START} - -{DIVIDER} - -{REPLACE_END} -```""" - -# Available models -AVAILABLE_MODELS = [ - { - "name": "Moonshot Kimi-K2", - "id": "moonshotai/Kimi-K2-Instruct", - "description": "Moonshot AI Kimi-K2-Instruct model for code generation and general tasks" - }, - { - "name": "DeepSeek V3", - "id": "deepseek-ai/DeepSeek-V3-0324", - "description": "DeepSeek V3 model for code generation" - }, - { - "name": "DeepSeek R1", - "id": "deepseek-ai/DeepSeek-R1-0528", - "description": "DeepSeek R1 model for code generation" - }, - { - "name": "ERNIE-4.5-VL", - "id": "baidu/ERNIE-4.5-VL-424B-A47B-Base-PT", - "description": "ERNIE-4.5-VL model for multimodal code generation with image support" - }, - { - "name": "MiniMax M1", - "id": "MiniMaxAI/MiniMax-M1-80k", - "description": "MiniMax M1 model for code generation and general tasks" - }, - { - "name": "Qwen3-235B-A22B", - "id": "Qwen/Qwen3-235B-A22B", - "description": "Qwen3-235B-A22B model for code generation and general tasks" - }, - { - "name": "SmolLM3-3B", - "id": "HuggingFaceTB/SmolLM3-3B", - "description": "SmolLM3-3B model for code generation and general tasks" - }, - { - "name": "GLM-4.5", - "id": "zai-org/GLM-4.5", - "description": "GLM-4.5 model with thinking capabilities for advanced code generation" - }, - { - "name": "GLM-4.1V-9B-Thinking", - "id": "THUDM/GLM-4.1V-9B-Thinking", - "description": "GLM-4.1V-9B-Thinking model for multimodal code generation with image support" - }, - { - "name": "Qwen3-235B-A22B-Instruct-2507", - "id": "Qwen/Qwen3-235B-A22B-Instruct-2507", - "description": "Qwen3-235B-A22B-Instruct-2507 model for code generation and general tasks" - }, - { - "name": "Qwen3-Coder-480B-A35B-Instruct", - "id": "Qwen/Qwen3-Coder-480B-A35B-Instruct", - "description": "Qwen3-Coder-480B-A35B-Instruct model for advanced code generation and programming tasks" - }, - { - "name": "Qwen3-32B", - "id": "Qwen/Qwen3-32B", - "description": "Qwen3-32B model for code generation and general tasks" - }, - { - "name": "Qwen3-4B-Instruct-2507", - "id": "Qwen/Qwen3-4B-Instruct-2507", - "description": "Qwen3-4B-Instruct-2507 model for code generation and general tasks" - }, - { - "name": "Qwen3-4B-Thinking-2507", - "id": "Qwen/Qwen3-4B-Thinking-2507", - "description": "Qwen3-4B-Thinking-2507 model with advanced reasoning capabilities for code generation and general tasks" - }, - { - "name": "Qwen3-235B-A22B-Thinking", - "id": "Qwen/Qwen3-235B-A22B-Thinking-2507", - "description": "Qwen3-235B-A22B-Thinking model with advanced reasoning capabilities" - }, - { - "name": "Qwen3-30B-A3B-Instruct-2507", - "id": "qwen3-30b-a3b-instruct-2507", - "description": "Qwen3-30B-A3B-Instruct model via Alibaba Cloud DashScope API" - }, - { - "name": "Qwen3-30B-A3B-Thinking-2507", - "id": "qwen3-30b-a3b-thinking-2507", - "description": "Qwen3-30B-A3B-Thinking model with advanced reasoning via Alibaba Cloud DashScope API" - }, - { - "name": "Qwen3-Coder-30B-A3B-Instruct", - "id": "qwen3-coder-30b-a3b-instruct", - "description": "Qwen3-Coder-30B-A3B-Instruct model for advanced code generation via Alibaba Cloud DashScope API" - }, - { - "name": "StepFun Step-3", - "id": "step-3", - "description": "StepFun Step-3 model - AI chat assistant by ι˜Άθ·ƒζ˜ŸθΎ° with multilingual capabilities" - }, - { - "name": "Codestral 2508", - "id": "codestral-2508", - "description": "Mistral Codestral model - specialized for code generation and programming tasks" - }, - { - "name": "GPT-OSS-120B", - "id": "openai/gpt-oss-120b", - "description": "OpenAI GPT-OSS-120B model for advanced code generation and general tasks" - }, - { - "name": "GPT-OSS-20B", - "id": "openai/gpt-oss-20b", - "description": "OpenAI GPT-OSS-20B model for code generation and general tasks" - }, - { - "name": "GPT-5", - "id": "gpt-5", - "description": "OpenAI GPT-5 model for advanced code generation and general tasks" - } -] - -DEMO_LIST = [ - { - "title": "Todo App", - "description": "Create a simple todo application with add, delete, and mark as complete functionality" - }, - { - "title": "Calculator", - "description": "Build a basic calculator with addition, subtraction, multiplication, and division" - }, - { - "title": "Chat Interface", - "description": "Build a chat interface with message history and user input" - }, - { - "title": "E-commerce Product Card", - "description": "Create a product card component for an e-commerce website" - }, - { - "title": "Login Form", - "description": "Build a responsive login form with validation" - }, - { - "title": "Dashboard Layout", - "description": "Create a dashboard layout with sidebar navigation and main content area" - }, - { - "title": "Data Table", - "description": "Build a data table with sorting and filtering capabilities" - }, - { - "title": "Image Gallery", - "description": "Create an image gallery with lightbox functionality and responsive grid layout" - }, - { - "title": "UI from Image", - "description": "Upload an image of a UI design and I'll generate the HTML/CSS code for it" - }, - { - "title": "Extract Text from Image", - "description": "Upload an image containing text and I'll extract and process the text content" - }, - { - "title": "Website Redesign", - "description": "Enter a website URL to extract its content and redesign it with a modern, responsive layout" - }, - { - "title": "Modify HTML", - "description": "After generating HTML, ask me to modify it with specific changes using search/replace format" - }, - { - "title": "Search/Replace Example", - "description": "Generate HTML first, then ask: 'Change the title to My New Title' or 'Add a blue background to the body'" - }, - { - "title": "Transformers.js App", - "description": "Create a transformers.js application with AI/ML functionality using the transformers.js library" - }, - { - "title": "Svelte App", - "description": "Create a modern Svelte application with TypeScript, Vite, and responsive design" - } -] - -# HF Inference Client -HF_TOKEN = os.getenv('HF_TOKEN') -if not HF_TOKEN: - raise RuntimeError("HF_TOKEN environment variable is not set. Please set it to your Hugging Face API token.") - -def get_inference_client(model_id, provider="auto"): - """Return an InferenceClient with provider based on model_id and user selection.""" - if model_id == "qwen3-30b-a3b-instruct-2507": - # Use DashScope OpenAI client - return OpenAI( - api_key=os.getenv("DASHSCOPE_API_KEY"), - base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", - ) - elif model_id == "qwen3-30b-a3b-thinking-2507": - # Use DashScope OpenAI client for Thinking model - return OpenAI( - api_key=os.getenv("DASHSCOPE_API_KEY"), - base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", - ) - elif model_id == "qwen3-coder-30b-a3b-instruct": - # Use DashScope OpenAI client for Coder model - return OpenAI( - api_key=os.getenv("DASHSCOPE_API_KEY"), - base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", - ) - elif model_id == "gpt-5": - # Use OpenAI client for GPT-5 model - return OpenAI( - api_key=os.getenv("OPENAI_API_KEY"), - ) - elif model_id == "step-3": - # Use StepFun API client for Step-3 model - return OpenAI( - api_key=os.getenv("STEP_API_KEY"), - base_url="https://api.stepfun.com/v1" - ) - elif model_id == "codestral-2508": - # Use Mistral client for Codestral model - return Mistral(api_key=os.getenv("MISTRAL_API_KEY")) - elif model_id == "openai/gpt-oss-120b": - provider = "cerebras" - elif model_id == "openai/gpt-oss-20b": - provider = "groq" - elif model_id == "moonshotai/Kimi-K2-Instruct": - provider = "groq" - elif model_id == "Qwen/Qwen3-235B-A22B": - provider = "cerebras" - elif model_id == "Qwen/Qwen3-235B-A22B-Instruct-2507": - provider = "cerebras" - elif model_id == "Qwen/Qwen3-32B": - provider = "cerebras" - elif model_id == "Qwen/Qwen3-235B-A22B-Thinking-2507": - provider = "cerebras" - elif model_id == "Qwen/Qwen3-Coder-480B-A35B-Instruct": - provider = "cerebras" - return InferenceClient( - provider=provider, - api_key=HF_TOKEN, - bill_to="huggingface" - ) - -# Type definitions -History = List[Tuple[str, str]] -Messages = List[Dict[str, str]] - -# Tavily Search Client -TAVILY_API_KEY = os.getenv('TAVILY_API_KEY') -tavily_client = None -if TAVILY_API_KEY: - try: - tavily_client = TavilyClient(api_key=TAVILY_API_KEY) - except Exception as e: - print(f"Failed to initialize Tavily client: {e}") - tavily_client = None - -def history_to_messages(history: History, system: str) -> Messages: - messages = [{'role': 'system', 'content': system}] - for h in history: - # Handle multimodal content in history - user_content = h[0] - if isinstance(user_content, list): - # Extract text from multimodal content - text_content = "" - for item in user_content: - if isinstance(item, dict) and item.get("type") == "text": - text_content += item.get("text", "") - user_content = text_content if text_content else str(user_content) - - messages.append({'role': 'user', 'content': user_content}) - messages.append({'role': 'assistant', 'content': h[1]}) - return messages - -def messages_to_history(messages: Messages) -> Tuple[str, History]: - assert messages[0]['role'] == 'system' - history = [] - for q, r in zip(messages[1::2], messages[2::2]): - # Extract text content from multimodal messages for history - user_content = q['content'] - if isinstance(user_content, list): - text_content = "" - for item in user_content: - if isinstance(item, dict) and item.get("type") == "text": - text_content += item.get("text", "") - user_content = text_content if text_content else str(user_content) - - history.append([user_content, r['content']]) - return history - -def history_to_chatbot_messages(history: History) -> List[Dict[str, str]]: - """Convert history tuples to chatbot message format""" - messages = [] - for user_msg, assistant_msg in history: - # Handle multimodal content - if isinstance(user_msg, list): - text_content = "" - for item in user_msg: - if isinstance(item, dict) and item.get("type") == "text": - text_content += item.get("text", "") - user_msg = text_content if text_content else str(user_msg) - - messages.append({"role": "user", "content": user_msg}) - messages.append({"role": "assistant", "content": assistant_msg}) - return messages - -def remove_code_block(text): - # Try to match code blocks with language markers - patterns = [ - r'```(?:html|HTML)\n([\s\S]+?)\n```', # Match ```html or ```HTML - r'```\n([\s\S]+?)\n```', # Match code blocks without language markers - r'```([\s\S]+?)```' # Match code blocks without line breaks - ] - for pattern in patterns: - match = re.search(pattern, text, re.DOTALL) - if match: - extracted = match.group(1).strip() - # Remove a leading language marker line (e.g., 'python') if present - if extracted.split('\n', 1)[0].strip().lower() in ['python', 'html', 'css', 'javascript', 'json', 'c', 'cpp', 'markdown', 'latex', 'jinja2', 'typescript', 'yaml', 'dockerfile', 'shell', 'r', 'sql', 'sql-mssql', 'sql-mysql', 'sql-mariadb', 'sql-sqlite', 'sql-cassandra', 'sql-plSQL', 'sql-hive', 'sql-pgsql', 'sql-gql', 'sql-gpsql', 'sql-sparksql', 'sql-esper']: - return extracted.split('\n', 1)[1] if '\n' in extracted else '' - return extracted - # If no code block is found, check if the entire text is HTML - if text.strip().startswith('') or text.strip().startswith(' 1 else '' - return text.strip() - -def parse_transformers_js_output(text): - """Parse transformers.js output and extract the three files (index.html, index.js, style.css)""" - files = { - 'index.html': '', - 'index.js': '', - 'style.css': '' - } - - # Multiple patterns to match the three code blocks with different variations - html_patterns = [ - r'```html\s*\n([\s\S]+?)\n```', - r'```htm\s*\n([\s\S]+?)\n```', - r'```\s*(?:index\.html|html)\s*\n([\s\S]+?)\n```' - ] - - js_patterns = [ - r'```javascript\s*\n([\s\S]+?)\n```', - r'```js\s*\n([\s\S]+?)\n```', - r'```\s*(?:index\.js|javascript)\s*\n([\s\S]+?)\n```' - ] - - css_patterns = [ - r'```css\s*\n([\s\S]+?)\n```', - r'```\s*(?:style\.css|css)\s*\n([\s\S]+?)\n```' - ] - - # Extract HTML content - for pattern in html_patterns: - html_match = re.search(pattern, text, re.IGNORECASE) - if html_match: - files['index.html'] = html_match.group(1).strip() - break - - # Extract JavaScript content - for pattern in js_patterns: - js_match = re.search(pattern, text, re.IGNORECASE) - if js_match: - files['index.js'] = js_match.group(1).strip() - break - - # Extract CSS content - for pattern in css_patterns: - css_match = re.search(pattern, text, re.IGNORECASE) - if css_match: - files['style.css'] = css_match.group(1).strip() - break - - # Fallback: support === index.html === format if any file is missing - if not (files['index.html'] and files['index.js'] and files['style.css']): - # Use regex to extract sections - html_fallback = re.search(r'===\s*index\.html\s*===\s*\n([\s\S]+?)(?=\n===|$)', text, re.IGNORECASE) - js_fallback = re.search(r'===\s*index\.js\s*===\s*\n([\s\S]+?)(?=\n===|$)', text, re.IGNORECASE) - css_fallback = re.search(r'===\s*style\.css\s*===\s*\n([\s\S]+?)(?=\n===|$)', text, re.IGNORECASE) - - if html_fallback: - files['index.html'] = html_fallback.group(1).strip() - if js_fallback: - files['index.js'] = js_fallback.group(1).strip() - if css_fallback: - files['style.css'] = css_fallback.group(1).strip() - - # Additional fallback: extract from numbered sections or file headers - if not (files['index.html'] and files['index.js'] and files['style.css']): - # Try patterns like "1. index.html:" or "**index.html**" - patterns = [ - (r'(?:^\d+\.\s*|^##\s*|^\*\*\s*)index\.html(?:\s*:|\*\*:?)\s*\n([\s\S]+?)(?=\n(?:\d+\.|##|\*\*|===)|$)', 'index.html'), - (r'(?:^\d+\.\s*|^##\s*|^\*\*\s*)index\.js(?:\s*:|\*\*:?)\s*\n([\s\S]+?)(?=\n(?:\d+\.|##|\*\*|===)|$)', 'index.js'), - (r'(?:^\d+\.\s*|^##\s*|^\*\*\s*)style\.css(?:\s*:|\*\*:?)\s*\n([\s\S]+?)(?=\n(?:\d+\.|##|\*\*|===)|$)', 'style.css') - ] - - for pattern, file_key in patterns: - if not files[file_key]: - match = re.search(pattern, text, re.IGNORECASE | re.MULTILINE) - if match: - # Clean up the content by removing any code block markers - content = match.group(1).strip() - content = re.sub(r'^```\w*\s*\n', '', content) - content = re.sub(r'\n```\s*$', '', content) - files[file_key] = content.strip() - - return files - -def format_transformers_js_output(files): - """Format the three files into a single display string""" - output = [] - output.append("=== index.html ===") - output.append(files['index.html']) - output.append("\n=== index.js ===") - output.append(files['index.js']) - output.append("\n=== style.css ===") - output.append(files['style.css']) - return '\n'.join(output) - -def build_transformers_inline_html(files: dict) -> str: - """Merge transformers.js three-file output into a single self-contained HTML document. - - - Inlines style.css into a " if css else "" - if style_tag: - if '' in doc.lower(): - # Preserve original casing by finding closing head case-insensitively - match = _re.search(r"", doc, flags=_re.IGNORECASE) - if match: - idx = match.start() - doc = doc[:idx] + style_tag + doc[idx:] - else: - # No head; insert at top of body - match = _re.search(r"]*>", doc, flags=_re.IGNORECASE) - if match: - idx = match.end() - doc = doc[:idx] + "\n" + style_tag + doc[idx:] - else: - # Append at beginning - doc = style_tag + doc - - # Inline JS: insert before - script_tag = f"" if js else "" - # Cleanup script to clear Cache Storage and IndexedDB on unload to free model weights - cleanup_tag = ( - "" - ) - if script_tag: - match = _re.search(r"", doc, flags=_re.IGNORECASE) - if match: - idx = match.start() - doc = doc[:idx] + script_tag + cleanup_tag + doc[idx:] - else: - # Append at end - doc = doc + script_tag + cleanup_tag - - return doc - -def send_transformers_to_sandbox(files: dict) -> str: - """Build a self-contained HTML document from transformers.js files and return an iframe preview.""" - merged_html = build_transformers_inline_html(files) - return send_to_sandbox(merged_html) - -def parse_svelte_output(text): - """Parse Svelte output to extract individual files""" - files = { - 'src/App.svelte': '', - 'src/app.css': '' - } - - import re - - # First try to extract using code block patterns - svelte_pattern = r'```svelte\s*\n([\s\S]+?)\n```' - css_pattern = r'```css\s*\n([\s\S]+?)\n```' - - # Extract svelte block for App.svelte - svelte_match = re.search(svelte_pattern, text, re.IGNORECASE) - css_match = re.search(css_pattern, text, re.IGNORECASE) - - if svelte_match: - files['src/App.svelte'] = svelte_match.group(1).strip() - if css_match: - files['src/app.css'] = css_match.group(1).strip() - - # Fallback: support === filename === format if any file is missing - if not (files['src/App.svelte'] and files['src/app.css']): - # Use regex to extract sections - app_svelte_fallback = re.search(r'===\s*src/App\.svelte\s*===\n([\s\S]+?)(?=\n===|$)', text, re.IGNORECASE) - app_css_fallback = re.search(r'===\s*src/app\.css\s*===\n([\s\S]+?)(?=\n===|$)', text, re.IGNORECASE) - - if app_svelte_fallback: - files['src/App.svelte'] = app_svelte_fallback.group(1).strip() - if app_css_fallback: - files['src/app.css'] = app_css_fallback.group(1).strip() - - return files - -def format_svelte_output(files): - """Format Svelte files into a single display string""" - output = [] - output.append("=== src/App.svelte ===") - output.append(files['src/App.svelte']) - output.append("\n=== src/app.css ===") - output.append(files['src/app.css']) - return '\n'.join(output) - -def history_render(history: History): - return gr.update(visible=True), history - -def clear_history(): - return [], [], None, "" # Empty lists for both tuple format and chatbot messages, None for file, empty string for website URL - -def update_image_input_visibility(model): - """Update image input visibility based on selected model""" - is_ernie_vl = model.get("id") == "baidu/ERNIE-4.5-VL-424B-A47B-Base-PT" - is_glm_vl = model.get("id") == "THUDM/GLM-4.1V-9B-Thinking" - return gr.update(visible=is_ernie_vl or is_glm_vl) - -def process_image_for_model(image): - """Convert image to base64 for model input""" - if image is None: - return None - - # Convert numpy array to PIL Image if needed - import io - import base64 - import numpy as np - from PIL import Image - - # Handle numpy array from Gradio - if isinstance(image, np.ndarray): - image = Image.fromarray(image) - - buffer = io.BytesIO() - image.save(buffer, format='PNG') - img_str = base64.b64encode(buffer.getvalue()).decode() - return f"data:image/png;base64,{img_str}" - -def generate_image_with_qwen(prompt: str, image_index: int = 0) -> str: - """Generate image using Qwen image model via Hugging Face InferenceClient with optimized data URL""" - try: - # Check if HF_TOKEN is available - if not os.getenv('HF_TOKEN'): - return "Error: HF_TOKEN environment variable is not set. Please set it to your Hugging Face API token." - - # Create InferenceClient for Qwen image generation - client = InferenceClient( - provider="auto", - api_key=os.getenv('HF_TOKEN'), - bill_to="huggingface", - ) - - # Generate image using Qwen/Qwen-Image model - image = client.text_to_image( - prompt, - model="Qwen/Qwen-Image", - ) - - # Resize image to reduce size while maintaining quality - max_size = 512 - if image.width > max_size or image.height > max_size: - image.thumbnail((max_size, max_size), Image.Resampling.LANCZOS) - - # Convert PIL Image to optimized base64 for HTML embedding - import io - import base64 - - buffer = io.BytesIO() - # Save as JPEG with compression for smaller file size - image.convert('RGB').save(buffer, format='JPEG', quality=85, optimize=True) - img_str = base64.b64encode(buffer.getvalue()).decode() - - # Return HTML img tag with optimized data URL - return f'{prompt}' - - except Exception as e: - print(f"Image generation error: {str(e)}") - return f"Error generating image: {str(e)}" - -def generate_image_to_image(input_image_data, prompt: str) -> str: - """Generate an image using image-to-image with FLUX.1-Kontext-dev via Hugging Face InferenceClient. - - Returns an HTML tag with optimized base64 JPEG data, similar to text-to-image output. - """ - try: - # Check token - if not os.getenv('HF_TOKEN'): - return "Error: HF_TOKEN environment variable is not set. Please set it to your Hugging Face API token." - - # Prepare client - client = InferenceClient( - provider="auto", - api_key=os.getenv('HF_TOKEN'), - bill_to="huggingface", - ) - - # Normalize input image to bytes - import io - from PIL import Image - try: - import numpy as np - except Exception: - np = None - - if hasattr(input_image_data, 'read'): - # File-like object - raw = input_image_data.read() - pil_image = Image.open(io.BytesIO(raw)) - elif hasattr(input_image_data, 'mode') and hasattr(input_image_data, 'size'): - # PIL Image - pil_image = input_image_data - elif np is not None and isinstance(input_image_data, np.ndarray): - pil_image = Image.fromarray(input_image_data) - elif isinstance(input_image_data, (bytes, bytearray)): - pil_image = Image.open(io.BytesIO(input_image_data)) - else: - # Fallback: try to convert via bytes - pil_image = Image.open(io.BytesIO(bytes(input_image_data))) - - # Ensure RGB - if pil_image.mode != 'RGB': - pil_image = pil_image.convert('RGB') - - buf = io.BytesIO() - pil_image.save(buf, format='PNG') - input_bytes = buf.getvalue() - - # Call image-to-image - image = client.image_to_image( - input_bytes, - prompt=prompt, - model="black-forest-labs/FLUX.1-Kontext-dev", - ) - - # Resize/optimize - max_size = 512 - if image.width > max_size or image.height > max_size: - image.thumbnail((max_size, max_size), Image.Resampling.LANCZOS) - - out_buf = io.BytesIO() - image.convert('RGB').save(out_buf, format='JPEG', quality=85, optimize=True) - - import base64 - img_str = base64.b64encode(out_buf.getvalue()).decode() - return f"\"{prompt}\"" - except Exception as e: - print(f"Image-to-image generation error: {str(e)}") - return f"Error generating image (image-to-image): {str(e)}" - -def extract_image_prompts_from_text(text: str, num_images_needed: int = 1) -> list: - """Extract image generation prompts from the full text based on number of images needed""" - # Use the entire text as the base prompt for image generation - # Clean up the text and create variations for the required number of images - - # Clean the text - cleaned_text = text.strip() - if not cleaned_text: - return [] - - # Create variations of the prompt for the required number of images - prompts = [] - - # Generate exactly the number of images needed - for i in range(num_images_needed): - if i == 0: - # First image: Use the full prompt as-is - prompts.append(cleaned_text) - elif i == 1: - # Second image: Add "visual representation" to make it more image-focused - prompts.append(f"Visual representation of {cleaned_text}") - elif i == 2: - # Third image: Add "illustration" to create a different style - prompts.append(f"Illustration of {cleaned_text}") - else: - # For additional images, use different variations - variations = [ - f"Digital art of {cleaned_text}", - f"Modern design of {cleaned_text}", - f"Professional illustration of {cleaned_text}", - f"Clean design of {cleaned_text}", - f"Beautiful visualization of {cleaned_text}", - f"Stylish representation of {cleaned_text}", - f"Contemporary design of {cleaned_text}", - f"Elegant illustration of {cleaned_text}" - ] - variation_index = (i - 3) % len(variations) - prompts.append(variations[variation_index]) - - return prompts - -def create_image_replacement_blocks(html_content: str, user_prompt: str) -> str: - """Create search/replace blocks to replace placeholder images with generated Qwen images""" - if not user_prompt: - return "" - - # Find existing image placeholders in the HTML first - import re - - # Common patterns for placeholder images - placeholder_patterns = [ - r']*src=["\'](?:placeholder|dummy|sample|example)[^"\']*["\'][^>]*>', - r']*src=["\']https?://via\.placeholder\.com[^"\']*["\'][^>]*>', - r']*src=["\']https?://picsum\.photos[^"\']*["\'][^>]*>', - r']*src=["\']https?://dummyimage\.com[^"\']*["\'][^>]*>', - r']*alt=["\'][^"\']*placeholder[^"\']*["\'][^>]*>', - r']*class=["\'][^"\']*placeholder[^"\']*["\'][^>]*>', - r']*id=["\'][^"\']*placeholder[^"\']*["\'][^>]*>', - r']*src=["\']data:image[^"\']*["\'][^>]*>', # Base64 images - r']*src=["\']#["\'][^>]*>', # Empty src - r']*src=["\']about:blank["\'][^>]*>', # About blank - ] - - # Find all placeholder images - placeholder_images = [] - for pattern in placeholder_patterns: - matches = re.findall(pattern, html_content, re.IGNORECASE) - placeholder_images.extend(matches) - - # If no placeholder images found, look for any img tags - if not placeholder_images: - img_pattern = r']*>' - placeholder_images = re.findall(img_pattern, html_content) - - # Also look for div elements that might be image placeholders - div_placeholder_patterns = [ - r']*class=["\'][^"\']*(?:image|img|photo|picture)[^"\']*["\'][^>]*>.*?', - r']*id=["\'][^"\']*(?:image|img|photo|picture)[^"\']*["\'][^>]*>.*?', - ] - - for pattern in div_placeholder_patterns: - matches = re.findall(pattern, html_content, re.IGNORECASE | re.DOTALL) - placeholder_images.extend(matches) - - # Count how many images we need to generate - num_images_needed = len(placeholder_images) - - if num_images_needed == 0: - return "" - - # Generate image prompts based on the number of images found - image_prompts = extract_image_prompts_from_text(user_prompt, num_images_needed) - - # Generate images for each prompt - generated_images = [] - for i, prompt in enumerate(image_prompts): - image_html = generate_image_with_qwen(prompt, i) - if not image_html.startswith("Error"): - generated_images.append((i, image_html)) - - if not generated_images: - return "" - - # Create search/replace blocks - replacement_blocks = [] - - for i, (prompt_index, generated_image) in enumerate(generated_images): - if i < len(placeholder_images): - # Replace existing placeholder - placeholder = placeholder_images[i] - # Clean up the placeholder for better matching - placeholder_clean = re.sub(r'\s+', ' ', placeholder.strip()) - - # Try multiple variations of the placeholder for better matching - placeholder_variations = [ - placeholder_clean, - placeholder_clean.replace('"', "'"), - placeholder_clean.replace("'", '"'), - re.sub(r'\s+', ' ', placeholder_clean), - placeholder_clean.replace(' ', ' '), - ] - - # Create a replacement block for each variation - for variation in placeholder_variations: - replacement_blocks.append(f"""{SEARCH_START} -{variation} -{DIVIDER} -{generated_image} -{REPLACE_END}""") - else: - # Add new image if we have more generated images than placeholders - # Find a good insertion point (after body tag or main content) - if '', html_content.find(' str: - """Create search/replace blocks that generate and insert ONLY ONE text-to-image result. - - Replaces the first detected placeholder; if none found, inserts one image near the top of . - """ - if not prompt or not prompt.strip(): - return "" - - import re - - # Detect placeholders similarly to the multi-image version - placeholder_patterns = [ - r']*src=["\'](?:placeholder|dummy|sample|example)[^"\']*["\'][^>]*>', - r']*src=["\']https?://via\.placeholder\.com[^"\']*["\'][^>]*>', - r']*src=["\']https?://picsum\.photos[^"\']*["\'][^>]*>', - r']*src=["\']https?://dummyimage\.com[^"\']*["\'][^>]*>', - r']*alt=["\'][^"\']*placeholder[^"\']*["\'][^>]*>', - r']*class=["\'][^"\']*placeholder[^"\']*["\'][^>]*>', - r']*id=["\'][^"\']*placeholder[^"\']*["\'][^>]*>', - r']*src=["\']data:image[^"\']*["\'][^>]*>', - r']*src=["\']#["\'][^>]*>', - r']*src=["\']about:blank["\'][^>]*>', - ] - - placeholder_images = [] - for pattern in placeholder_patterns: - matches = re.findall(pattern, html_content, re.IGNORECASE) - if matches: - placeholder_images.extend(matches) - - # Fallback to any if no placeholders - if not placeholder_images: - img_pattern = r']*>' - placeholder_images = re.findall(img_pattern, html_content) - - # Generate a single image - image_html = generate_image_with_qwen(prompt, 0) - if image_html.startswith("Error"): - return "" - - # Replace first placeholder if present - if placeholder_images: - placeholder = placeholder_images[0] - placeholder_clean = re.sub(r'\s+', ' ', placeholder.strip()) - placeholder_variations = [ - placeholder_clean, - placeholder_clean.replace('"', "'"), - placeholder_clean.replace("'", '"'), - re.sub(r'\s+', ' ', placeholder_clean), - placeholder_clean.replace(' ', ' '), - ] - blocks = [] - for variation in placeholder_variations: - blocks.append(f"""{SEARCH_START} -{variation} -{DIVIDER} -{image_html} -{REPLACE_END}""") - return '\n\n'.join(blocks) - - # Otherwise insert after - if '', html_content.find(', just append - return f"{SEARCH_START}\n\n{DIVIDER}\n{image_html}\n{REPLACE_END}" - -def create_image_replacement_blocks_from_input_image(html_content: str, user_prompt: str, input_image_data, max_images: int = 1) -> str: - """Create search/replace blocks using image-to-image generation with a provided input image. - - Mirrors placeholder detection from create_image_replacement_blocks but uses generate_image_to_image. - """ - if not user_prompt: - return "" - import re - - placeholder_patterns = [ - r']*src=["\'](?:placeholder|dummy|sample|example)[^"\']*["\'][^>]*>', - r']*src=["\']https?://via\.placeholder\.com[^"\']*["\'][^>]*>', - r']*src=["\']https?://picsum\.photos[^"\']*["\'][^>]*>', - r']*src=["\']https?://dummyimage\.com[^"\']*["\'][^>]*>', - r']*alt=["\'][^"\']*placeholder[^"\']*["\'][^>]*>', - r']*class=["\'][^"\']*placeholder[^"\']*["\'][^>]*>', - r']*id=["\'][^"\']*placeholder[^"\']*["\'][^>]*>', - r']*src=["\']data:image[^"\']*["\'][^>]*>', - r']*src=["\']#["\'][^>]*>', - r']*src=["\']about:blank["\'][^>]*>', - ] - - placeholder_images = [] - for pattern in placeholder_patterns: - matches = re.findall(pattern, html_content, re.IGNORECASE) - placeholder_images.extend(matches) - - if not placeholder_images: - img_pattern = r']*>' - placeholder_images = re.findall(img_pattern, html_content) - - div_placeholder_patterns = [ - r']*class=["\'][^"\']*(?:image|img|photo|picture)[^"\']*["\'][^>]*>.*?', - r']*id=["\'][^"\']*(?:image|img|photo|picture)[^"\']*["\'][^>]*>.*?', - ] - for pattern in div_placeholder_patterns: - matches = re.findall(pattern, html_content, re.IGNORECASE | re.DOTALL) - placeholder_images.extend(matches) - - num_images_needed = len(placeholder_images) - num_to_replace = min(num_images_needed, max(0, int(max_images))) - if num_images_needed == 0: - # No placeholders; generate one image to append (only if at least one upload is present) - if num_to_replace <= 0: - return "" - prompts = extract_image_prompts_from_text(user_prompt, 1) - if not prompts: - return "" - image_html = generate_image_to_image(input_image_data, prompts[0]) - if image_html.startswith("Error"): - return "" - return f"{SEARCH_START}\n\n{DIVIDER}\n
{image_html}
\n{REPLACE_END}" - - if num_to_replace <= 0: - return "" - image_prompts = extract_image_prompts_from_text(user_prompt, num_to_replace) - - generated_images = [] - for i, prompt in enumerate(image_prompts): - image_html = generate_image_to_image(input_image_data, prompt) - if not image_html.startswith("Error"): - generated_images.append((i, image_html)) - - if not generated_images: - return "" - - replacement_blocks = [] - for i, (prompt_index, generated_image) in enumerate(generated_images): - if i < num_to_replace and i < len(placeholder_images): - placeholder = placeholder_images[i] - placeholder_clean = re.sub(r'\s+', ' ', placeholder.strip()) - placeholder_variations = [ - placeholder_clean, - placeholder_clean.replace('"', "'"), - placeholder_clean.replace("'", '"'), - re.sub(r'\s+', ' ', placeholder_clean), - placeholder_clean.replace(' ', ' '), - ] - for variation in placeholder_variations: - replacement_blocks.append(f"""{SEARCH_START} -{variation} -{DIVIDER} -{generated_image} -{REPLACE_END}""") - # Do not insert additional images beyond the uploaded count - - return '\n\n'.join(replacement_blocks) - -def apply_generated_images_to_html(html_content: str, user_prompt: str, enable_text_to_image: bool, enable_image_to_image: bool, input_image_data, image_to_image_prompt: str | None = None, text_to_image_prompt: str | None = None) -> str: - """Apply text-to-image and/or image-to-image replacements to HTML content. - - If both toggles are enabled, text-to-image replacements run first, then image-to-image. - """ - result = html_content - try: - # If an input image is provided and image-to-image is enabled, we only replace one image - # and skip text-to-image to satisfy the requirement to replace exactly the number of uploaded images. - if enable_image_to_image and input_image_data is not None and (result.strip().startswith('') or result.strip().startswith('') or result.strip().startswith(' str: - """Apply search/replace changes to content (HTML, Python, etc.)""" - if not changes_text.strip(): - return original_content - - # Split the changes text into individual search/replace blocks - blocks = [] - current_block = "" - lines = changes_text.split('\n') - - for line in lines: - if line.strip() == SEARCH_START: - if current_block.strip(): - blocks.append(current_block.strip()) - current_block = line + '\n' - elif line.strip() == REPLACE_END: - current_block += line + '\n' - blocks.append(current_block.strip()) - current_block = "" - else: - current_block += line + '\n' - - if current_block.strip(): - blocks.append(current_block.strip()) - - modified_content = original_content - - for block in blocks: - if not block.strip(): - continue - - # Parse the search/replace block - lines = block.split('\n') - search_lines = [] - replace_lines = [] - in_search = False - in_replace = False - - for line in lines: - if line.strip() == SEARCH_START: - in_search = True - in_replace = False - elif line.strip() == DIVIDER: - in_search = False - in_replace = True - elif line.strip() == REPLACE_END: - in_replace = False - elif in_search: - search_lines.append(line) - elif in_replace: - replace_lines.append(line) - - # Apply the search/replace - if search_lines: - search_text = '\n'.join(search_lines).strip() - replace_text = '\n'.join(replace_lines).strip() - - if search_text in modified_content: - modified_content = modified_content.replace(search_text, replace_text) - else: - print(f"Warning: Search text not found in content: {search_text[:100]}...") - - return modified_content - -def apply_transformers_js_search_replace_changes(original_formatted_content: str, changes_text: str) -> str: - """Apply search/replace changes to transformers.js formatted content (three files)""" - if not changes_text.strip(): - return original_formatted_content - - # Parse the original formatted content to get the three files - files = parse_transformers_js_output(original_formatted_content) - - # Split the changes text into individual search/replace blocks - blocks = [] - current_block = "" - lines = changes_text.split('\n') - - for line in lines: - if line.strip() == SEARCH_START: - if current_block.strip(): - blocks.append(current_block.strip()) - current_block = line + '\n' - elif line.strip() == REPLACE_END: - current_block += line + '\n' - blocks.append(current_block.strip()) - current_block = "" - else: - current_block += line + '\n' - - if current_block.strip(): - blocks.append(current_block.strip()) - - # Process each block and apply changes to the appropriate file - for block in blocks: - if not block.strip(): - continue - - # Parse the search/replace block - lines = block.split('\n') - search_lines = [] - replace_lines = [] - in_search = False - in_replace = False - target_file = None - - for line in lines: - if line.strip() == SEARCH_START: - in_search = True - in_replace = False - elif line.strip() == DIVIDER: - in_search = False - in_replace = True - elif line.strip() == REPLACE_END: - in_replace = False - elif in_search: - search_lines.append(line) - elif in_replace: - replace_lines.append(line) - - # Determine which file this change targets based on the search content - if search_lines: - search_text = '\n'.join(search_lines).strip() - replace_text = '\n'.join(replace_lines).strip() - - # Check which file contains the search text - if search_text in files['index.html']: - target_file = 'index.html' - elif search_text in files['index.js']: - target_file = 'index.js' - elif search_text in files['style.css']: - target_file = 'style.css' - - # Apply the change to the target file - if target_file and search_text in files[target_file]: - files[target_file] = files[target_file].replace(search_text, replace_text) - else: - print(f"Warning: Search text not found in any transformers.js file: {search_text[:100]}...") - - # Reformat the modified files - return format_transformers_js_output(files) - -# Updated for faster Tavily search and closer prompt usage -# Uses 'advanced' search_depth and auto_parameters=True for speed and relevance - -def perform_web_search(query: str, max_results: int = 5, include_domains=None, exclude_domains=None) -> str: - """Perform web search using Tavily with default parameters""" - if not tavily_client: - return "Web search is not available. Please set the TAVILY_API_KEY environment variable." - - try: - # Use Tavily defaults with advanced search depth for better results - search_params = { - "search_depth": "advanced", - "max_results": min(max(1, max_results), 20) - } - if include_domains is not None: - search_params["include_domains"] = include_domains - if exclude_domains is not None: - search_params["exclude_domains"] = exclude_domains - - response = tavily_client.search(query, **search_params) - - search_results = [] - for result in response.get('results', []): - title = result.get('title', 'No title') - url = result.get('url', 'No URL') - content = result.get('content', 'No content') - search_results.append(f"Title: {title}\nURL: {url}\nContent: {content}\n") - - if search_results: - return "Web Search Results:\n\n" + "\n---\n".join(search_results) - else: - return "No search results found." - - except Exception as e: - return f"Search error: {str(e)}" - -def enhance_query_with_search(query: str, enable_search: bool) -> str: - """Enhance the query with web search results if search is enabled""" - if not enable_search or not tavily_client: - return query - - # Perform search to get relevant information - search_results = perform_web_search(query) - - # Combine original query with search results - enhanced_query = f"""Original Query: {query} - -{search_results} - -Please use the search results above to help create the requested application with the most up-to-date information and best practices.""" - - return enhanced_query - -def send_to_sandbox(code): - # Add a wrapper to inject necessary permissions and ensure full HTML - wrapped_code = f""" - - - - - - - - - {code} - - - """ - encoded_html = base64.b64encode(wrapped_code.encode('utf-8')).decode('utf-8') - data_uri = f"data:text/html;charset=utf-8;base64,{encoded_html}" - iframe = f'' - return iframe - -def is_streamlit_code(code: str) -> bool: - """Heuristic check to determine if Python code is a Streamlit app.""" - if not code: - return False - lowered = code.lower() - return ("import streamlit" in lowered) or ("from streamlit" in lowered) or ("st." in code and "streamlit" in lowered) - -def send_streamlit_to_stlite(code: str) -> str: - """Render Streamlit code using stlite inside a sandboxed iframe for preview.""" - # Build an HTML document that loads stlite and mounts the Streamlit app defined inline - html_doc = ( - """ - - - - - - Streamlit Preview - - - - - - -""" - + (code or "") - + """ - - - -""" - ) - encoded_html = base64.b64encode(html_doc.encode('utf-8')).decode('utf-8') - data_uri = f"data:text/html;charset=utf-8;base64,{encoded_html}" - iframe = f'' - return iframe - -def is_gradio_code(code: str) -> bool: - """Heuristic check to determine if Python code is a Gradio app.""" - if not code: - return False - lowered = code.lower() - return ( - "import gradio" in lowered - or "from gradio" in lowered - or "gr.Interface(" in code - or "gr.Blocks(" in code - ) - -def send_gradio_to_lite(code: str) -> str: - """Render Gradio code using gradio-lite inside a sandboxed iframe for preview.""" - html_doc = ( - """ - - - - - - Gradio Preview - - - - - - -""" - + (code or "") - + """ - - - -""" - ) - encoded_html = base64.b64encode(html_doc.encode('utf-8')).decode('utf-8') - data_uri = f"data:text/html;charset=utf-8;base64,{encoded_html}" - iframe = f'' - return iframe - -def demo_card_click(e: gr.EventData): - try: - # Get the index from the event data - if hasattr(e, '_data') and e._data: - # Try different ways to get the index - if 'index' in e._data: - index = e._data['index'] - elif 'component' in e._data and 'index' in e._data['component']: - index = e._data['component']['index'] - elif 'target' in e._data and 'index' in e._data['target']: - index = e._data['target']['index'] - else: - # If we can't get the index, try to extract it from the card data - index = 0 - else: - index = 0 - - # Ensure index is within bounds - if index >= len(DEMO_LIST): - index = 0 - - return DEMO_LIST[index]['description'] - except (KeyError, IndexError, AttributeError) as e: - # Return the first demo description as fallback - return DEMO_LIST[0]['description'] - -def extract_text_from_image(image_path): - """Extract text from image using OCR""" - try: - # Check if tesseract is available - try: - pytesseract.get_tesseract_version() - except Exception: - return "Error: Tesseract OCR is not installed. Please install Tesseract to extract text from images. See install_tesseract.md for instructions." - - # Read image using OpenCV - image = cv2.imread(image_path) - if image is None: - return "Error: Could not read image file" - - # Convert to RGB (OpenCV uses BGR) - image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) - - # Preprocess image for better OCR results - # Convert to grayscale - gray = cv2.cvtColor(image_rgb, cv2.COLOR_RGB2GRAY) - - # Apply thresholding to get binary image - _, binary = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) - - # Extract text using pytesseract - text = pytesseract.image_to_string(binary, config='--psm 6') - - return text.strip() if text.strip() else "No text found in image" - - except Exception as e: - return f"Error extracting text from image: {e}" - -def extract_text_from_file(file_path): - if not file_path: - return "" - mime, _ = mimetypes.guess_type(file_path) - ext = os.path.splitext(file_path)[1].lower() - try: - if ext == ".pdf": - with open(file_path, "rb") as f: - reader = PyPDF2.PdfReader(f) - return "\n".join(page.extract_text() or "" for page in reader.pages) - elif ext in [".txt", ".md"]: - with open(file_path, "r", encoding="utf-8") as f: - return f.read() - elif ext == ".csv": - with open(file_path, "r", encoding="utf-8") as f: - return f.read() - elif ext == ".docx": - doc = docx.Document(file_path) - return "\n".join([para.text for para in doc.paragraphs]) - elif ext.lower() in [".jpg", ".jpeg", ".png", ".bmp", ".tiff", ".tif", ".gif", ".webp"]: - return extract_text_from_image(file_path) - else: - return "" - except Exception as e: - return f"Error extracting text: {e}" - -def extract_website_content(url: str) -> str: - """Extract HTML code and content from a website URL""" - try: - # Validate URL - parsed_url = urlparse(url) - if not parsed_url.scheme: - url = "https://" + url - parsed_url = urlparse(url) - - if not parsed_url.netloc: - return "Error: Invalid URL provided" - - # Set comprehensive headers to mimic a real browser request - headers = { - 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', - 'Accept-Language': 'en-US,en;q=0.9', - 'Accept-Encoding': 'gzip, deflate, br', - 'DNT': '1', - 'Connection': 'keep-alive', - 'Upgrade-Insecure-Requests': '1', - 'Sec-Fetch-Dest': 'document', - 'Sec-Fetch-Mode': 'navigate', - 'Sec-Fetch-Site': 'none', - 'Sec-Fetch-User': '?1', - 'Cache-Control': 'max-age=0' - } - - # Create a session to maintain cookies and handle redirects - session = requests.Session() - session.headers.update(headers) - - # Make the request with retry logic - max_retries = 3 - for attempt in range(max_retries): - try: - response = session.get(url, timeout=15, allow_redirects=True) - response.raise_for_status() - break - except requests.exceptions.HTTPError as e: - if e.response.status_code == 403 and attempt < max_retries - 1: - # Try with different User-Agent on 403 - session.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' - continue - else: - raise - - # Get the raw HTML content with proper encoding - try: - # Try to get the content with automatic encoding detection - response.encoding = response.apparent_encoding - raw_html = response.text - except: - # Fallback to UTF-8 if encoding detection fails - raw_html = response.content.decode('utf-8', errors='ignore') - - # Debug: Check if we got valid HTML - if not raw_html.strip().startswith(' 10: - print(f"Warning: This site has {len(script_tags)} script tags - it may be a JavaScript-heavy site") - print("The content might be loaded dynamically and not available in the initial HTML") - - # Extract title - title = soup.find('title') - title_text = title.get_text().strip() if title else "No title found" - - # Extract meta description - meta_desc = soup.find('meta', attrs={'name': 'description'}) - description = meta_desc.get('content', '') if meta_desc else "" - - # Extract main content areas for analysis - content_sections = [] - main_selectors = [ - 'main', 'article', '.content', '.main-content', '.post-content', - '#content', '#main', '.entry-content', '.post-body' - ] - - for selector in main_selectors: - elements = soup.select(selector) - for element in elements: - text = element.get_text().strip() - if len(text) > 100: # Only include substantial content - content_sections.append(text) - - # Extract navigation links for analysis - nav_links = [] - nav_elements = soup.find_all(['nav', 'header']) - for nav in nav_elements: - links = nav.find_all('a') - for link in links: - link_text = link.get_text().strip() - link_href = link.get('href', '') - if link_text and link_href: - nav_links.append(f"{link_text}: {link_href}") - - # Extract and fix image URLs in the HTML - img_elements = soup.find_all('img') - for img in img_elements: - src = img.get('src', '') - if src: - # Handle different URL formats - if src.startswith('//'): - # Protocol-relative URL - absolute_src = 'https:' + src - img['src'] = absolute_src - elif src.startswith('/'): - # Root-relative URL - absolute_src = urljoin(url, src) - img['src'] = absolute_src - elif not src.startswith(('http://', 'https://')): - # Relative URL - absolute_src = urljoin(url, src) - img['src'] = absolute_src - # If it's already absolute, keep it as is - - # Also check for data-src (lazy loading) and other common attributes - data_src = img.get('data-src', '') - if data_src and not src: - # Use data-src if src is empty - if data_src.startswith('//'): - absolute_data_src = 'https:' + data_src - img['src'] = absolute_data_src - elif data_src.startswith('/'): - absolute_data_src = urljoin(url, data_src) - img['src'] = absolute_data_src - elif not data_src.startswith(('http://', 'https://')): - absolute_data_src = urljoin(url, data_src) - img['src'] = absolute_data_src - else: - img['src'] = data_src - - # Also fix background image URLs in style attributes - elements_with_style = soup.find_all(attrs={'style': True}) - for element in elements_with_style: - style_attr = element.get('style', '') - # Find and replace relative URLs in background-image - import re - bg_pattern = r'background-image:\s*url\(["\']?([^"\']+)["\']?\)' - matches = re.findall(bg_pattern, style_attr, re.IGNORECASE) - for match in matches: - if match: - if match.startswith('//'): - absolute_bg = 'https:' + match - style_attr = style_attr.replace(match, absolute_bg) - elif match.startswith('/'): - absolute_bg = urljoin(url, match) - style_attr = style_attr.replace(match, absolute_bg) - elif not match.startswith(('http://', 'https://')): - absolute_bg = urljoin(url, match) - style_attr = style_attr.replace(match, absolute_bg) - element['style'] = style_attr - - # Fix background images in