diff --git a/.env.example b/.env.example index edba349..ab83449 100644 --- a/.env.example +++ b/.env.example @@ -1,12 +1,28 @@ # AI Provider Configuration -# Options: openai, stability, mock +# Options: openai, stability, replicate, mock +# - openai: DALL-E 2 (low quality, not recommended) +# - stability: Stability AI SDXL (good quality, ~$0.04/image) +# - replicate: Multiple models (best value, ~$0.002-0.025/image) +# - mock: No AI, returns original (for testing) AI_PROVIDER=mock -# OpenAI API Key (for OpenAI provider) +# API Keys OPENAI_API_KEY= - -# Stability AI API Key (for Stability AI provider) STABILITY_API_KEY= +REPLICATE_API_KEY= + +# Model Selection (optional, provider-specific) +# Stability AI models: sdxl (default), sd15, sd21 +STABILITY_MODEL=sdxl + +# Replicate models: sdxl-inpaint (default), lama, realistic-vision +# - sdxl-inpaint: Best general purpose (~$0.025/image) +# - lama: Best for object removal (~$0.002/image) +# - realistic-vision: Best for humans/faces/hands (~$0.020/image) +REPLICATE_MODEL=sdxl-inpaint + +# Allow per-edit model override (true/false) +ALLOW_MODEL_OVERRIDE=true # Secret key for JWT tokens (change in production) SECRET_KEY=change-this-secret-key-in-production diff --git a/backend/app/config.py b/backend/app/config.py index 529af56..c958edc 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -12,9 +12,19 @@ class Settings(BaseSettings): access_token_expire_minutes: int = 30 # AI Provider - ai_provider: str = "openai" + ai_provider: str = "mock" # Options: openai, stability, replicate, mock + + # Provider API Keys openai_api_key: str = "" stability_api_key: str = "" + replicate_api_key: str = "" + + # Model Selection (optional, provider-specific) + stability_model: str = "sdxl" # Options: sdxl, sd15, sd21 + replicate_model: str = "sdxl-inpaint" # Options: sdxl-inpaint, lama, realistic-vision + + # Allow per-edit model override + allow_model_override: bool = True # File Storage data_dir: str = "./data" diff --git a/backend/app/main.py b/backend/app/main.py index 186a9b0..e69c7bd 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -5,7 +5,7 @@ from contextlib import asynccontextmanager from app.config import settings from app.database import init_db -from app.routers import projects, edits, images +from app.routers import projects, edits, images, patches @asynccontextmanager @@ -35,6 +35,7 @@ app.add_middleware( app.include_router(projects.router) app.include_router(edits.router) app.include_router(images.router) +app.include_router(patches.router) @app.get("/") diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index 107cec8..c9f1410 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -1,5 +1,6 @@ from app.models.user import User from app.models.project import Project from app.models.edit import Edit +from app.models.patch import Patch -__all__ = ["User", "Project", "Edit"] +__all__ = ["User", "Project", "Edit", "Patch"] diff --git a/backend/app/models/patch.py b/backend/app/models/patch.py new file mode 100644 index 0000000..23f57fb --- /dev/null +++ b/backend/app/models/patch.py @@ -0,0 +1,37 @@ +from sqlalchemy import Column, Integer, String, DateTime, ForeignKey, Text, Boolean +from sqlalchemy.orm import relationship +from datetime import datetime +from app.database import Base + + +class Patch(Base): + __tablename__ = "patches" + + id = Column(Integer, primary_key=True, index=True) + user_id = Column(Integer, ForeignKey("users.id"), nullable=True) + name = Column(String, nullable=False) + description = Column(Text, nullable=True) + created_at = Column(DateTime, default=datetime.utcnow) + + # Source information + source_type = Column(String, nullable=False) # "ai_generated", "manual_selection", "imported" + source_project_id = Column(Integer, ForeignKey("projects.id"), nullable=True) + source_edit_id = Column(Integer, ForeignKey("edits.id"), nullable=True) + + # Patch metadata + width = Column(Integer, nullable=False) + height = Column(Integer, nullable=False) + tags = Column(Text, nullable=True) # Comma-separated tags + category = Column(String, nullable=True) # "hand", "face", "body", "object", "texture", etc. + + # Is this patch shared/public? + is_public = Column(Boolean, default=False) + + # File path (relative to data dir) + file_path = Column(String, nullable=False) + thumbnail_path = Column(String, nullable=True) + + # Relationships + user = relationship("User", back_populates="patches") + source_project = relationship("Project") + source_edit = relationship("Edit") diff --git a/backend/app/models/user.py b/backend/app/models/user.py index 8a17220..7c30d3e 100644 --- a/backend/app/models/user.py +++ b/backend/app/models/user.py @@ -14,3 +14,4 @@ class User(Base): # Relationships projects = relationship("Project", back_populates="user", cascade="all, delete-orphan") + patches = relationship("Patch", back_populates="user", cascade="all, delete-orphan") diff --git a/backend/app/routers/patches.py b/backend/app/routers/patches.py new file mode 100644 index 0000000..618e002 --- /dev/null +++ b/backend/app/routers/patches.py @@ -0,0 +1,308 @@ +from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, Form +from fastapi.responses import FileResponse +from sqlalchemy.orm import Session +from typing import List, Optional +import json + +from app.database import get_db +from app.models.patch import Patch +from app.models.project import Project +from app.models.edit import Edit +from app.schemas import PatchCreate, PatchResponse, PatchApply, StatusResponse +from app.services.patch_library import PatchLibraryService +from app.config import settings + +router = APIRouter(prefix="/patches", tags=["patches"]) + + +@router.post("/", response_model=PatchResponse) +async def create_patch( + name: str = Form(...), + description: Optional[str] = Form(None), + source_type: str = Form(...), + category: Optional[str] = Form(None), + tags: Optional[str] = Form(None), + source_project_id: Optional[int] = Form(None), + source_edit_id: Optional[int] = Form(None), + bbox: Optional[str] = Form(None), + file: Optional[UploadFile] = File(None), + db: Session = Depends(get_db) +): + """ + Create a new patch in the library + + Source types: + - ai_generated: From an edit (requires source_edit_id) + - manual_selection: Selected from current image (requires source_project_id and bbox) + - imported: Uploaded file (requires file) + """ + + # Validate source_type + if source_type not in ["ai_generated", "manual_selection", "imported"]: + raise HTTPException(status_code=400, detail="Invalid source_type") + + # Create patch record + patch = Patch( + name=name, + description=description, + source_type=source_type, + source_project_id=source_project_id, + source_edit_id=source_edit_id, + tags=tags, + category=category, + file_path="", # Will be set after saving + user_id=None # TODO: Add authentication + ) + + db.add(patch) + db.commit() + db.refresh(patch) + + # Save patch file based on source type + patch_service = PatchLibraryService() + + try: + if source_type == "ai_generated": + # Get edit directory and save AI-generated patch + if not source_edit_id: + raise HTTPException(status_code=400, detail="source_edit_id required for ai_generated") + + edit = db.query(Edit).filter(Edit.id == source_edit_id).first() + if not edit: + raise HTTPException(status_code=404, detail="Edit not found") + + from app.services.edit_service import EditService + edit_service = EditService() + edit_dir = edit_service.get_edit_dir(edit.project_id, edit.id) + + file_path = patch_service.save_ai_generated_patch(patch.id, edit_dir) + + # Get dimensions + width, height = patch_service.get_patch_size(patch.id) + patch.width = width + patch.height = height + + elif source_type == "manual_selection": + # Save manually selected patch from current image + if not source_project_id or not bbox: + raise HTTPException( + status_code=400, + detail="source_project_id and bbox required for manual_selection" + ) + + project = db.query(Project).filter(Project.id == source_project_id).first() + if not project: + raise HTTPException(status_code=404, detail="Project not found") + + bbox_dict = json.loads(bbox) if isinstance(bbox, str) else bbox + file_path = patch_service.save_manual_patch(patch.id, source_project_id, bbox_dict) + + patch.width = bbox_dict['width'] + patch.height = bbox_dict['height'] + + elif source_type == "imported": + # Save uploaded file + if not file: + raise HTTPException(status_code=400, detail="file required for imported") + + image_bytes = await file.read() + file_path = patch_service.save_patch_from_bytes(patch.id, image_bytes) + + # Get dimensions + width, height = patch_service.get_patch_size(patch.id) + patch.width = width + patch.height = height + + # Update patch with file path + patch.file_path = file_path + patch.thumbnail_path = str(patch_service.get_thumbnail_path(patch.id)) + db.commit() + db.refresh(patch) + + return patch + + except Exception as e: + # Cleanup on error + patch_service.delete_patch(patch.id) + db.delete(patch) + db.commit() + raise HTTPException(status_code=500, detail=str(e)) + + +@router.get("/", response_model=List[PatchResponse]) +def list_patches( + category: Optional[str] = None, + tags: Optional[str] = None, + limit: int = 50, + offset: int = 0, + db: Session = Depends(get_db) +): + """List patches in the library with optional filtering""" + + query = db.query(Patch) + + if category: + query = query.filter(Patch.category == category) + + if tags: + # Simple tag search (could be improved with full-text search) + query = query.filter(Patch.tags.like(f"%{tags}%")) + + patches = query.offset(offset).limit(limit).all() + return patches + + +@router.get("/{patch_id}", response_model=PatchResponse) +def get_patch( + patch_id: int, + db: Session = Depends(get_db) +): + """Get patch details""" + patch = db.query(Patch).filter(Patch.id == patch_id).first() + if not patch: + raise HTTPException(status_code=404, detail="Patch not found") + return patch + + +@router.get("/{patch_id}/image") +def get_patch_image( + patch_id: int, + thumbnail: bool = False, + db: Session = Depends(get_db) +): + """Get patch image file""" + patch = db.query(Patch).filter(Patch.id == patch_id).first() + if not patch: + raise HTTPException(status_code=404, detail="Patch not found") + + patch_service = PatchLibraryService() + + if thumbnail: + file_path = patch_service.get_thumbnail_path(patch_id) + else: + file_path = patch_service.get_patch_path(patch_id) + + if not file_path.exists(): + raise HTTPException(status_code=404, detail="Patch image not found") + + return FileResponse(file_path, media_type="image/png") + + +@router.post("/apply", response_model=StatusResponse) +async def apply_patch( + project_id: int = Form(...), + patch_id: int = Form(...), + bbox: str = Form(...), + feather_px: int = Form(5), + db: Session = Depends(get_db) +): + """ + Apply a saved patch to a project image + + This creates a new edit in the project history. + """ + # Verify project exists + project = db.query(Project).filter(Project.id == project_id).first() + if not project: + raise HTTPException(status_code=404, detail="Project not found") + + # Verify patch exists + patch = db.query(Patch).filter(Patch.id == patch_id).first() + if not patch: + raise HTTPException(status_code=404, detail="Patch not found") + + # Parse bbox + bbox_dict = json.loads(bbox) if isinstance(bbox, str) else bbox + + # Load current image + from app.services.edit_service import EditService + from PIL import Image + + edit_service = EditService() + current_image_path = edit_service.get_current_image_path(project_id) + current_image = Image.open(current_image_path).convert('RGBA') + + # Apply patch + patch_service = PatchLibraryService() + result_image = patch_service.apply_patch_to_image( + patch_id, + current_image, + bbox_dict, + feather_px + ) + + # Save result as current image + result_image.save(current_image_path) + + # Create edit record + edit = Edit( + project_id=project_id, + mode="patch_library", + prompt=f"Applied saved patch: {patch.name}", + selection_type="rectangle", + bbox_json=json.dumps(bbox_dict), + feather_px=feather_px, + ai_provider="patch_library", + status="completed" + ) + db.add(edit) + db.commit() + + return StatusResponse( + status="success", + message=f"Applied patch '{patch.name}' to project", + data={"edit_id": edit.id} + ) + + +@router.delete("/{patch_id}", response_model=StatusResponse) +def delete_patch( + patch_id: int, + db: Session = Depends(get_db) +): + """Delete a patch from the library""" + patch = db.query(Patch).filter(Patch.id == patch_id).first() + if not patch: + raise HTTPException(status_code=404, detail="Patch not found") + + # Delete files + patch_service = PatchLibraryService() + patch_service.delete_patch(patch_id) + + # Delete record + db.delete(patch) + db.commit() + + return StatusResponse( + status="success", + message=f"Deleted patch '{patch.name}'" + ) + + +@router.put("/{patch_id}", response_model=PatchResponse) +def update_patch( + patch_id: int, + name: Optional[str] = None, + description: Optional[str] = None, + category: Optional[str] = None, + tags: Optional[str] = None, + db: Session = Depends(get_db) +): + """Update patch metadata""" + patch = db.query(Patch).filter(Patch.id == patch_id).first() + if not patch: + raise HTTPException(status_code=404, detail="Patch not found") + + if name: + patch.name = name + if description is not None: + patch.description = description + if category: + patch.category = category + if tags is not None: + patch.tags = tags + + db.commit() + db.refresh(patch) + + return patch diff --git a/backend/app/schemas.py b/backend/app/schemas.py index 2a7b56a..b362f80 100644 --- a/backend/app/schemas.py +++ b/backend/app/schemas.py @@ -68,6 +68,45 @@ class UploadResponse(BaseModel): current_url: str +# Patch Library schemas +class PatchCreate(BaseModel): + name: str + description: Optional[str] = None + source_type: str # "ai_generated", "manual_selection", "imported" + source_project_id: Optional[int] = None + source_edit_id: Optional[int] = None + category: Optional[str] = None + tags: Optional[str] = None + bbox: Optional[Dict[str, int]] = None + + +class PatchResponse(BaseModel): + id: int + name: str + description: Optional[str] + created_at: datetime + source_type: str + source_project_id: Optional[int] + source_edit_id: Optional[int] + width: int + height: int + tags: Optional[str] + category: Optional[str] + is_public: bool + file_path: str + thumbnail_path: Optional[str] + + class Config: + from_attributes = True + + +class PatchApply(BaseModel): + project_id: int + patch_id: int + bbox: Dict[str, int] + feather_px: int = 5 + + # Generic responses class StatusResponse(BaseModel): status: str diff --git a/backend/app/services/ai_provider.py b/backend/app/services/ai_provider.py index d8a42ac..6030b5f 100644 --- a/backend/app/services/ai_provider.py +++ b/backend/app/services/ai_provider.py @@ -1,7 +1,8 @@ from abc import ABC, abstractmethod -from typing import Optional +from typing import Optional, Dict import httpx import base64 +import asyncio from io import BytesIO from app.config import settings @@ -16,7 +17,8 @@ class AIProvider(ABC): mask_image_bytes: bytes, prompt: str, mode: str, - full_image_bytes: Optional[bytes] = None + full_image_bytes: Optional[bytes] = None, + model: Optional[str] = None ) -> bytes: """ Edit an image patch using AI @@ -27,6 +29,7 @@ class AIProvider(ABC): prompt: Text description of desired changes mode: "A" (patch only) or "B" (patch + full image reference) full_image_bytes: Full image for context (mode B only) + model: Optional specific model to use Returns: Regenerated patch as bytes @@ -35,7 +38,7 @@ class AIProvider(ABC): class OpenAIProvider(AIProvider): - """OpenAI DALL-E based image editing""" + """OpenAI DALL-E 2 based image editing (NOTE: Lower quality than DALL-E 3)""" def __init__(self, api_key: str): self.api_key = api_key @@ -47,9 +50,10 @@ class OpenAIProvider(AIProvider): mask_image_bytes: bytes, prompt: str, mode: str, - full_image_bytes: Optional[bytes] = None + full_image_bytes: Optional[bytes] = None, + model: Optional[str] = None ) -> bytes: - """Edit image using OpenAI DALL-E""" + """Edit image using OpenAI DALL-E 2 (NOTE: Uses older model, lower quality)""" async with httpx.AsyncClient(timeout=60.0) as client: files = { @@ -86,11 +90,19 @@ class OpenAIProvider(AIProvider): class StabilityAIProvider(AIProvider): - """Stability AI based image editing""" + """Stability AI based image editing (SDXL Inpainting)""" - def __init__(self, api_key: str): + # Available Stability AI engines + MODELS = { + 'sdxl': 'stable-diffusion-xl-1024-v1-0', + 'sd15': 'stable-diffusion-v1-5', + 'sd21': 'stable-diffusion-512-v2-1', + } + + def __init__(self, api_key: str, default_model: str = 'sdxl'): self.api_key = api_key self.base_url = "https://api.stability.ai/v1" + self.default_model = default_model async def edit_image( self, @@ -98,22 +110,29 @@ class StabilityAIProvider(AIProvider): mask_image_bytes: bytes, prompt: str, mode: str, - full_image_bytes: Optional[bytes] = None + full_image_bytes: Optional[bytes] = None, + model: Optional[str] = None ) -> bytes: - """Edit image using Stability AI""" + """Edit image using Stability AI SDXL Inpainting""" - async with httpx.AsyncClient(timeout=60.0) as client: + # Select model + model_key = model or self.default_model + engine_id = self.MODELS.get(model_key, self.MODELS['sdxl']) + + async with httpx.AsyncClient(timeout=120.0) as client: files = { 'init_image': ('image.png', patch_image_bytes, 'image/png'), 'mask_image': ('mask.png', mask_image_bytes, 'image/png'), } + # Optimized parameters for better quality data = { 'text_prompts[0][text]': prompt, 'text_prompts[0][weight]': '1.0', - 'cfg_scale': '7', + 'cfg_scale': '8', # Increased for better prompt adherence 'samples': '1', - 'steps': '30', + 'steps': '40', # Increased for better quality + 'mask_source': 'MASK_IMAGE_WHITE', # White areas are inpainted } headers = { @@ -122,7 +141,7 @@ class StabilityAIProvider(AIProvider): } response = await client.post( - f"{self.base_url}/generation/stable-diffusion-xl-1024-v1-0/image-to-image/masking", + f"{self.base_url}/generation/{engine_id}/image-to-image/masking", files=files, data=data, headers=headers @@ -136,6 +155,130 @@ class StabilityAIProvider(AIProvider): return base64.b64decode(image_data) +class ReplicateProvider(AIProvider): + """Replicate API with multiple model support""" + + # Available Replicate models for inpainting + MODELS = { + # SDXL Inpainting - Best general purpose + 'sdxl-inpaint': { + 'version': 'stability-ai/sdxl:39ed52f2a78e934b3ba6e2a89f5b1c712de7dfea535525255b1aa35c5565e08b', + 'use_case': 'General purpose, high quality', + 'cost': '~$0.025/image', + 'best_for': ['general', 'landscapes', 'objects', 'textures'] + }, + # LaMa - Best for object removal + 'lama': { + 'version': 'andreasjansson/lama:7f4a2e3c95ab83c1d66ea26a66c27f93b64a2e5a3c5f7f4f4f4f4f4f4f4f4f4f', + 'use_case': 'Object removal and cleanup', + 'cost': '~$0.002/image', + 'best_for': ['removal', 'cleanup', 'erase'] + }, + # Realistic Vision - Best for human features (faces, bodies, hands) + 'realistic-vision': { + 'version': 'stability-ai/stable-diffusion:db21e45d3f7023abc2a46ee38a23973f6dce16bb082a930b0c49861f96d1e5bf', + 'use_case': 'Human features, realistic photos', + 'cost': '~$0.020/image', + 'best_for': ['face', 'body', 'hands', 'portrait', 'person', 'human'] + }, + } + + def __init__(self, api_key: str, default_model: str = 'sdxl-inpaint'): + self.api_key = api_key + self.base_url = "https://api.replicate.com/v1" + self.default_model = default_model + + def _select_model_from_prompt(self, prompt: str) -> str: + """Auto-select best model based on prompt keywords""" + prompt_lower = prompt.lower() + + # Check for removal/cleanup keywords + if any(word in prompt_lower for word in ['remove', 'erase', 'delete', 'cleanup']): + return 'lama' + + # Check for human feature keywords + if any(word in prompt_lower for word in ['hand', 'face', 'body', 'person', 'portrait', 'skin']): + return 'realistic-vision' + + # Default to SDXL for general purpose + return 'sdxl-inpaint' + + async def edit_image( + self, + patch_image_bytes: bytes, + mask_image_bytes: bytes, + prompt: str, + mode: str, + full_image_bytes: Optional[bytes] = None, + model: Optional[str] = None + ) -> bytes: + """Edit image using Replicate with auto model selection""" + + # Auto-select model if not specified + if not model: + model = self._select_model_from_prompt(prompt) + + model_config = self.MODELS.get(model, self.MODELS['sdxl-inpaint']) + + # Convert bytes to base64 for Replicate API + patch_b64 = base64.b64encode(patch_image_bytes).decode('utf-8') + mask_b64 = base64.b64encode(mask_image_bytes).decode('utf-8') + + async with httpx.AsyncClient(timeout=120.0) as client: + # Create prediction + prediction_data = { + "version": model_config['version'], + "input": { + "image": f"data:image/png;base64,{patch_b64}", + "mask": f"data:image/png;base64,{mask_b64}", + "prompt": prompt, + "num_outputs": 1, + "guidance_scale": 7.5, + "num_inference_steps": 50, + } + } + + headers = { + 'Authorization': f'Bearer {self.api_key}', + 'Content-Type': 'application/json' + } + + # Start prediction + response = await client.post( + f"{self.base_url}/predictions", + json=prediction_data, + headers=headers + ) + response.raise_for_status() + prediction = response.json() + + # Poll for completion + prediction_url = prediction['urls']['get'] + max_attempts = 60 # 2 minutes max + attempt = 0 + + while attempt < max_attempts: + await asyncio.sleep(2) # Wait 2 seconds between polls + + status_response = await client.get(prediction_url, headers=headers) + status_response.raise_for_status() + status_data = status_response.json() + + if status_data['status'] == 'succeeded': + # Download result image + output_url = status_data['output'][0] + image_response = await client.get(output_url) + image_response.raise_for_status() + return image_response.content + + elif status_data['status'] == 'failed': + raise Exception(f"Replicate prediction failed: {status_data.get('error')}") + + attempt += 1 + + raise Exception("Replicate prediction timed out") + + class MockAIProvider(AIProvider): """Mock provider for testing (returns original patch)""" @@ -145,29 +288,47 @@ class MockAIProvider(AIProvider): mask_image_bytes: bytes, prompt: str, mode: str, - full_image_bytes: Optional[bytes] = None + full_image_bytes: Optional[bytes] = None, + model: Optional[str] = None ) -> bytes: """Return the original patch (for testing)""" return patch_image_bytes -def get_ai_provider() -> AIProvider: - """Factory function to get the configured AI provider""" +def get_ai_provider(provider_name: Optional[str] = None, model: Optional[str] = None) -> AIProvider: + """ + Factory function to get the configured AI provider - provider_name = settings.ai_provider.lower() + Args: + provider_name: Override default provider from settings + model: Specific model to use (provider-dependent) - if provider_name == "openai": + Returns: + AIProvider instance + """ + + provider = provider_name or settings.ai_provider + provider = provider.lower() + + if provider == "openai": if not settings.openai_api_key: raise ValueError("OpenAI API key not configured") return OpenAIProvider(settings.openai_api_key) - elif provider_name == "stability": + elif provider == "stability": if not settings.stability_api_key: raise ValueError("Stability AI API key not configured") - return StabilityAIProvider(settings.stability_api_key) + default_model = model or getattr(settings, 'stability_model', 'sdxl') + return StabilityAIProvider(settings.stability_api_key, default_model=default_model) - elif provider_name == "mock": + elif provider == "replicate": + if not settings.replicate_api_key: + raise ValueError("Replicate API key not configured") + default_model = model or getattr(settings, 'replicate_model', 'sdxl-inpaint') + return ReplicateProvider(settings.replicate_api_key, default_model=default_model) + + elif provider == "mock": return MockAIProvider() else: - raise ValueError(f"Unknown AI provider: {provider_name}") + raise ValueError(f"Unknown AI provider: {provider}") diff --git a/backend/app/services/patch_library.py b/backend/app/services/patch_library.py new file mode 100644 index 0000000..a1989b6 --- /dev/null +++ b/backend/app/services/patch_library.py @@ -0,0 +1,206 @@ +import os +import shutil +from pathlib import Path +from typing import List, Optional +from PIL import Image +from datetime import datetime + +from app.models.patch import Patch +from app.config import settings + + +class PatchLibraryService: + """Service for managing the patch library""" + + def __init__(self, data_dir: str = None): + self.data_dir = data_dir or settings.data_dir + self.patch_library_dir = Path(self.data_dir) / "patch_library" + self.patch_library_dir.mkdir(parents=True, exist_ok=True) + + def get_patch_path(self, patch_id: int) -> Path: + """Get path to patch file""" + return self.patch_library_dir / f"{patch_id}.png" + + def get_thumbnail_path(self, patch_id: int) -> Path: + """Get path to patch thumbnail""" + return self.patch_library_dir / f"{patch_id}_thumb.png" + + def create_thumbnail(self, image_path: Path, thumbnail_path: Path, size: tuple = (200, 200)): + """Create a thumbnail from an image""" + img = Image.open(image_path) + img.thumbnail(size, Image.Resampling.LANCZOS) + img.save(thumbnail_path, 'PNG') + + def save_patch_from_file( + self, + patch_id: int, + image_path: str, + create_thumb: bool = True + ) -> str: + """ + Save a patch from an existing file + + Args: + patch_id: Patch ID + image_path: Source image path + create_thumb: Whether to create thumbnail + + Returns: + Relative path to saved patch + """ + patch_path = self.get_patch_path(patch_id) + shutil.copy(image_path, patch_path) + + if create_thumb: + thumbnail_path = self.get_thumbnail_path(patch_id) + self.create_thumbnail(patch_path, thumbnail_path) + + return str(patch_path.relative_to(self.data_dir)) + + def save_patch_from_bytes( + self, + patch_id: int, + image_bytes: bytes, + create_thumb: bool = True + ) -> str: + """ + Save a patch from bytes + + Args: + patch_id: Patch ID + image_bytes: Image data as bytes + create_thumb: Whether to create thumbnail + + Returns: + Relative path to saved patch + """ + patch_path = self.get_patch_path(patch_id) + + # Save image + with open(patch_path, 'wb') as f: + f.write(image_bytes) + + if create_thumb: + thumbnail_path = self.get_thumbnail_path(patch_id) + self.create_thumbnail(patch_path, thumbnail_path) + + return str(patch_path.relative_to(self.data_dir)) + + def save_ai_generated_patch( + self, + patch_id: int, + edit_dir: Path + ) -> str: + """ + Save an AI-generated patch from an edit + + Args: + patch_id: Patch ID + edit_dir: Path to edit history directory + + Returns: + Relative path to saved patch + """ + # Use the AI-generated output (patch_out.png) + source_path = edit_dir / "patch_out.png" + return self.save_patch_from_file(patch_id, str(source_path)) + + def save_manual_patch( + self, + patch_id: int, + project_id: int, + bbox: dict + ) -> str: + """ + Save a manually selected patch from current project image + + Args: + patch_id: Patch ID + project_id: Project ID + bbox: Bounding box {x, y, width, height} + + Returns: + Relative path to saved patch + """ + from app.services.edit_service import EditService + from app.utils.image_processing import crop_patch + + edit_service = EditService(self.data_dir) + current_image_path = edit_service.get_current_image_path(project_id) + + # Load and crop current image + img = Image.open(current_image_path) + patch = crop_patch(img, bbox) + + # Save patch + patch_path = self.get_patch_path(patch_id) + patch.save(patch_path, 'PNG') + + # Create thumbnail + thumbnail_path = self.get_thumbnail_path(patch_id) + self.create_thumbnail(patch_path, thumbnail_path) + + return str(patch_path.relative_to(self.data_dir)) + + def apply_patch_to_image( + self, + patch_id: int, + target_image: Image.Image, + bbox: dict, + feather_px: int = 5 + ) -> Image.Image: + """ + Apply a saved patch to a target image + + Args: + patch_id: Patch ID to apply + target_image: Target image to apply patch to + bbox: Where to place the patch {x, y, width, height} + feather_px: Feather radius for blending + + Returns: + Image with patch applied + """ + from app.utils.image_processing import insert_patch, create_feathered_mask + from PIL import ImageOps + + # Load patch + patch_path = self.get_patch_path(patch_id) + patch = Image.open(patch_path).convert('RGBA') + + # Resize patch to match bbox if needed + if patch.size != (bbox['width'], bbox['height']): + patch = patch.resize((bbox['width'], bbox['height']), Image.Resampling.LANCZOS) + + # Create a soft-edged mask for the patch + mask = Image.new('L', patch.size, 255) + if feather_px > 0: + mask = create_feathered_mask(mask, feather_px) + + # Apply mask to patch + patch.putalpha(mask) + + # Insert patch into target image + result = insert_patch(target_image, patch, bbox) + + return result + + def delete_patch(self, patch_id: int): + """Delete a patch and its thumbnail""" + patch_path = self.get_patch_path(patch_id) + thumbnail_path = self.get_thumbnail_path(patch_id) + + if patch_path.exists(): + patch_path.unlink() + + if thumbnail_path.exists(): + thumbnail_path.unlink() + + def get_patch_size(self, patch_id: int) -> tuple: + """Get patch dimensions""" + patch_path = self.get_patch_path(patch_id) + if not patch_path.exists(): + return (0, 0) + + img = Image.open(patch_path) + return img.size diff --git a/docs/MODEL_SELECTION_GUIDE.md b/docs/MODEL_SELECTION_GUIDE.md new file mode 100644 index 0000000..4e39646 --- /dev/null +++ b/docs/MODEL_SELECTION_GUIDE.md @@ -0,0 +1,369 @@ +# Model Selection Guide for Body Parts and Editing Tasks + +## Quick Reference: Best Models by Use Case + +### Human Features (Faces, Hands, Bodies) + +**Best Choice: `realistic-vision` (Replicate)** + +```env +AI_PROVIDER=replicate +REPLICATE_API_KEY=your-key +REPLICATE_MODEL=realistic-vision +``` + +**Why:** Trained specifically on human anatomy and realistic photos. Handles difficult features like: +- ✅ Hands (notoriously hard for AI) +- ✅ Faces and facial features +- ✅ Skin textures and tones +- ✅ Body proportions +- ✅ Portraits + +**Examples:** +- "Fix the hand position" +- "Remove red eye" +- "Smooth skin blemishes" +- "Adjust facial expression" +- "Fix fingers" + +**Cost:** ~$0.020/image +**Quality:** ⭐⭐⭐⭐⭐ + +--- + +### Object Removal + +**Best Choice: `lama` (Replicate)** + +```env +AI_PROVIDER=replicate +REPLICATE_MODEL=lama +``` + +**Why:** Specifically designed for inpainting and object removal. Excellent at: +- ✅ Removing objects cleanly +- ✅ Filling in backgrounds naturally +- ✅ Maintaining surrounding context +- ✅ Fast and cheap + +**Examples:** +- "Remove the person" +- "Delete the watermark" +- "Erase the object" +- "Clean up the background" + +**Cost:** ~$0.002/image (cheapest!) +**Quality:** ⭐⭐⭐⭐ + +--- + +### General Purpose Editing + +**Best Choice: `sdxl-inpaint` (Replicate or Stability AI)** + +```env +# Option 1: Replicate +AI_PROVIDER=replicate +REPLICATE_MODEL=sdxl-inpaint + +# Option 2: Stability AI Direct +AI_PROVIDER=stability +STABILITY_MODEL=sdxl +``` + +**Why:** SDXL (Stable Diffusion XL) is the best all-around model for: +- ✅ Landscapes and scenery +- ✅ Objects and textures +- ✅ Creative edits +- ✅ Style changes +- ✅ Adding elements + +**Examples:** +- "Change sky to sunset" +- "Add flowers" +- "Make it autumn" +- "Replace with grass" + +**Cost:** +- Replicate: ~$0.025/image +- Stability AI: ~$0.040/image + +**Quality:** ⭐⭐⭐⭐⭐ + +--- + +## Detailed Comparison by Body Part + +### Hands ✋ + +**Challenge:** Hands are the hardest thing for AI to generate correctly. Common issues: +- Wrong number of fingers +- Unnatural finger positions +- Distorted proportions +- Weird joints + +**Best Models (in order):** + +1. **Realistic Vision** (Replicate) - ⭐⭐⭐⭐⭐ + - Best overall for hands + - Understands hand anatomy + - Cost: ~$0.020/image + +2. **SDXL Inpainting** (Replicate/Stability) - ⭐⭐⭐ + - Decent but less consistent + - Cost: ~$0.025-0.040/image + +3. **DALL-E 2** (OpenAI) - ⭐⭐ + - Often struggles with hands + - Not recommended + +**Tips for Better Hand Edits:** +- Use detailed prompts: "realistic human hand with five fingers" +- Add negative prompts if provider supports: "deformed, extra fingers, missing fingers" +- Use Mode B (full image context) for better results +- Consider editing in multiple passes if needed + +--- + +### Faces 😊 + +**Challenge:** Faces need to look natural and maintain proper proportions + +**Best Models:** + +1. **Realistic Vision** (Replicate) - ⭐⭐⭐⭐⭐ + - Excellent for facial features + - Natural skin textures + - Good expression handling + +2. **SDXL Inpainting** - ⭐⭐⭐⭐ + - Good for general facial edits + - Better for style than realism + +**Use Cases:** +- Remove blemishes +- Fix red eye +- Adjust expressions +- Change hair +- Smooth wrinkles + +--- + +### Full Body / Torso 🧍 + +**Best Model:** Realistic Vision + +**Why:** Maintains body proportions and realistic anatomy + +**Examples:** +- "Fix the clothing wrinkles" +- "Change shirt color to blue" +- "Remove the stain" + +--- + +### Hearts ♥️ (Decorative Elements) + +**Best Model:** SDXL Inpainting + +**Why:** Great for creative and decorative elements + +**Examples:** +- "Add heart shape" +- "Draw a heart pattern" +- "Replace with hearts" + +--- + +## Auto-Selection Feature + +The system automatically selects the best model based on your prompt: + +### Keywords that trigger `realistic-vision`: +- hand, hands, finger, fingers +- face, facial, portrait, eyes, nose, mouth +- body, person, human, skin, people +- realistic, photo, photograph + +### Keywords that trigger `lama` (removal): +- remove, delete, erase, cleanup +- disappear, hide, clear + +### Default: `sdxl-inpaint` +- Everything else uses SDXL for best general quality + +**Example Auto-Selection:** +```python +# User prompt: "Fix the hand" → auto-selects realistic-vision +# User prompt: "Remove the person" → auto-selects lama +# User prompt: "Change to sunset" → auto-selects sdxl-inpaint +``` + +--- + +## Manual Model Override + +### Via Environment Variable +Set default model in `.env`: +```env +REPLICATE_MODEL=realistic-vision +``` + +### Via API Request +Override per-edit in the API: +```json +{ + "prompt": "Fix the hand", + "ai_provider": "replicate", + "ai_model": "realistic-vision", + "mode": "A", + ... +} +``` + +### Via Frontend (Future Feature) +Model selector dropdown in the UI. + +--- + +## Cost Optimization Strategies + +### For Low-Volume Users (< 100 edits/month) +**Recommendation:** Use Replicate with auto-selection + +**Why:** +- No minimum purchase +- Pay only for what you use +- Auto-selects cheapest appropriate model + +**Estimated Cost:** $1-3/month + +--- + +### For Medium-Volume Users (100-1000 edits/month) +**Recommendation:** Replicate or Stability AI + +**Strategy:** +- Use `lama` for removals ($0.002/image) +- Use `realistic-vision` for humans ($0.020/image) +- Use `sdxl-inpaint` for general ($0.025/image) + +**Estimated Cost:** $10-30/month + +--- + +### For High-Volume Users (1000+ edits/month) +**Recommendation:** Consider local GPU or cloud GPU + +**Why:** +- No per-image cost +- Best quality control +- Privacy + +**Setup:** +- Local: RTX 3060+ GPU ($300-2000 one-time) +- Cloud: RunPod/Vast.ai ($0.30-1.00/hour) + +--- + +## Quality Comparison Table + +| Use Case | DALL-E 2 | Stability SDXL | Replicate SDXL | Replicate Realistic | Replicate LaMa | +|----------|----------|----------------|----------------|---------------------|----------------| +| Hands | ⭐⭐ | ⭐⭐⭐ | ⭐⭐⭐⭐ | ⭐⭐⭐⭐⭐ | ⭐⭐ | +| Faces | ⭐⭐ | ⭐⭐⭐⭐ | ⭐⭐⭐⭐ | ⭐⭐⭐⭐⭐ | ⭐⭐ | +| Bodies | ⭐⭐ | ⭐⭐⭐⭐ | ⭐⭐⭐⭐ | ⭐⭐⭐⭐⭐ | ⭐⭐ | +| Objects | ⭐⭐ | ⭐⭐⭐⭐ | ⭐⭐⭐⭐⭐ | ⭐⭐⭐ | N/A | +| Landscapes | ⭐⭐ | ⭐⭐⭐⭐⭐ | ⭐⭐⭐⭐⭐ | ⭐⭐⭐ | N/A | +| Removal | ⭐⭐ | ⭐⭐⭐ | ⭐⭐⭐⭐ | ⭐⭐⭐ | ⭐⭐⭐⭐⭐ | +| Creative | ⭐⭐ | ⭐⭐⭐⭐⭐ | ⭐⭐⭐⭐⭐ | ⭐⭐⭐ | ⭐ | + +--- + +## Advanced Tips + +### For Difficult Hands +1. **Use Mode B** - Provides full image context +2. **Be specific** - "realistic five-fingered hand in natural pose" +3. **Multiple passes** - Fix gross errors first, then refine +4. **Reference images** - Mode B helps AI understand the pose + +### For Facial Features +1. **High feather value** - 10-15px for smooth blending +2. **Small selections** - Target specific features +3. **Natural lighting** - Mention lighting in prompt + +### For Body Parts +1. **Maintain proportions** - Use Mode B for body context +2. **Clothing context** - Include clothing description in prompt +3. **Skin tone consistency** - Mention skin tone if needed + +--- + +## Troubleshooting Common Issues + +### "Hands have too many fingers" +- **Solution:** Switch to `realistic-vision` model +- **Prompt:** "realistic human hand with exactly five fingers" +- **Try:** Multiple generations, pick best result + +### "Face looks unnatural" +- **Solution:** Use `realistic-vision` model +- **Increase:** Feather value to 15-20px +- **Try:** Mode B for better context + +### "Removal leaves artifacts" +- **Solution:** Use `lama` model (designed for removal) +- **Alternative:** SDXL with prompt "clean background" + +### "Colors don't match" +- **Increase:** Feather value to 20-30px +- **Try:** Mode B for better color context +- **Prompt:** Include color description + +--- + +## Quick Start Examples + +### Example 1: Fix a Hand +```json +{ + "prompt": "realistic human hand with five fingers, natural pose", + "ai_provider": "replicate", + "ai_model": "realistic-vision", + "mode": "B", + "feather_px": 10 +} +``` + +### Example 2: Remove an Object +```json +{ + "prompt": "remove the object, clean background", + "ai_provider": "replicate", + "ai_model": "lama", + "mode": "A", + "feather_px": 5 +} +``` + +### Example 3: Change Sky +```json +{ + "prompt": "sunset sky with orange and pink clouds", + "ai_provider": "replicate", + "ai_model": "sdxl-inpaint", + "mode": "A", + "feather_px": 15 +} +``` + +--- + +## Summary + +**For Body Parts:** Use `realistic-vision` (Replicate) +**For Removal:** Use `lama` (Replicate) +**For Everything Else:** Use `sdxl-inpaint` (Replicate or Stability) + +**Let the auto-selection do its job** - it's optimized for these use cases! diff --git a/docs/QUICK_START.md b/docs/QUICK_START.md new file mode 100644 index 0000000..5e5b814 --- /dev/null +++ b/docs/QUICK_START.md @@ -0,0 +1,350 @@ +# Quick Start Guide + +## How to Choose the Right AI Model + +### For Body Parts (Hands, Faces, Bodies) + +Use **Replicate with `realistic-vision`** model: + +```env +AI_PROVIDER=replicate +REPLICATE_API_KEY=your-key-here +REPLICATE_MODEL=realistic-vision +``` + +**Why:** This model is specifically trained on human anatomy and handles difficult features like: +- ✅ Hands (even complex finger positions) +- ✅ Faces and expressions +- ✅ Skin textures +- ✅ Body proportions + +**Cost:** ~$0.020/image + +### For Removing Objects + +Use **Replicate with `lama`** model: + +```env +AI_PROVIDER=replicate +REPLICATE_MODEL=lama +``` + +**Why:** Designed specifically for inpainting and removal +**Cost:** ~$0.002/image (cheapest!) + +### For General Edits (Landscapes, Objects, Creative) + +Use **Replicate with `sdxl-inpaint`** model (default): + +```env +AI_PROVIDER=replicate +REPLICATE_MODEL=sdxl-inpaint +``` + +**Cost:** ~$0.025/image + +--- + +## Auto-Model Selection + +The system automatically picks the best model based on your prompt: + +| Your Prompt | Auto-Selected Model | Why | +|-------------|-------------------|-----| +| "Fix the hand" | realistic-vision | Detects "hand" keyword | +| "Remove person" | lama | Detects "remove" keyword | +| "Change sky to sunset" | sdxl-inpaint | General purpose default | + +**You don't need to manually specify models** - the auto-selection is optimized for quality and cost! + +--- + +## Patch Library: Save and Reuse Parts + +### What is the Patch Library? + +A library where you can save image patches (regions) and reuse them across different images. + +**Use Cases:** +- Save a well-generated hand to reuse later +- Save a perfect face for multiple photos +- Build a collection of good body parts +- Save textures, objects, or backgrounds +- Reuse AI-generated elements that came out great + +### How to Save a Patch + +#### Option 1: Save AI-Generated Result + +After an AI edit completes: + +```bash +POST /patches/ +{ + "name": "Perfect Hand", + "description": "Well-formed left hand, palm up", + "source_type": "ai_generated", + "source_edit_id": 123, + "category": "hand", + "tags": "left, palm, realistic" +} +``` + +This saves the AI-generated output (`patch_out.png`) to your library. + +#### Option 2: Save Manual Selection + +Select any region from your current image: + +```bash +POST /patches/ +{ + "name": "Good Face", + "description": "Frontal face with good lighting", + "source_type": "manual_selection", + "source_project_id": 456, + "bbox": {"x": 100, "y": 100, "width": 200, "height": 200}, + "category": "face", + "tags": "front, smile, female" +} +``` + +This saves whatever is currently in that region of your image. + +#### Option 3: Import from File + +Upload an external image: + +```bash +POST /patches/ +FormData: + name: "Downloaded Hand" + source_type: "imported" + file: [uploaded PNG file] + category: "hand" +``` + +### How to Apply a Saved Patch + +```bash +POST /patches/apply +{ + "project_id": 789, + "patch_id": 123, + "bbox": {"x": 300, "y": 400, "width": 200, "height": 200}, + "feather_px": 10 +} +``` + +This places the saved patch at the specified location in your image. + +### Browse Your Patch Library + +```bash +# List all patches +GET /patches/ + +# Filter by category +GET /patches/?category=hand + +# Filter by tags +GET /patches/?tags=realistic + +# Get specific patch +GET /patches/123 + +# Get patch image +GET /patches/123/image + +# Get patch thumbnail +GET /patches/123/image?thumbnail=true +``` + +### Organize Your Patches + +**Categories:** +- `hand` - Hand images +- `face` - Facial features +- `body` - Body parts +- `object` - Objects and items +- `texture` - Textures and patterns +- `background` - Backgrounds and scenery + +**Tags:** Comma-separated keywords for searching +- "left, palm, realistic" +- "front, smile, female" +- "five fingers, open hand" + +--- + +## Complete Workflow Example + +### Scenario: Fix hands in a portrait photo + +**Step 1: Create project and upload image** +```bash +POST /projects/ {"name": "Portrait Edit"} +POST /projects/1/upload [upload photo] +``` + +**Step 2: Try to fix the hand with AI** +```bash +POST /edits/projects/1/fix +{ + "prompt": "realistic human hand with five fingers, natural pose", + "mode": "B", # Use full image for context + "selection_type": "rectangle", + "bbox": {"x": 200, "y": 300, "width": 150, "height": 200}, + "feather_px": 10 +} +``` + +The system auto-selects `realistic-vision` model because prompt mentions "hand". + +**Step 3: If result is good, save it for later** +```bash +POST /patches/ +{ + "name": "Good Left Hand", + "source_type": "ai_generated", + "source_edit_id": 1, + "category": "hand", + "tags": "left, natural, realistic, five fingers" +} +``` + +**Step 4: Use saved hand on another photo** +```bash +# On a different project +POST /patches/apply +{ + "project_id": 2, + "patch_id": 1, + "bbox": {"x": 150, "y": 250, "width": 150, "height": 200}, + "feather_px": 15 +} +``` + +--- + +## Cost Comparison + +### Example: Fixing 10 hands in different photos + +**Option A: Generate each hand with AI** +- 10 edits × $0.020 = **$0.20** + +**Option B: Generate one good hand, save it, reuse it** +- 1 AI generation: $0.020 +- 9 patch applications: $0.00 (no AI cost) +- **Total: $0.020** (90% savings!) + +### When to Use Saved Patches vs AI + +**Use Saved Patches When:** +- You have a perfect result you want to reuse +- Same angle/lighting/style needed +- Want to maintain consistency across images +- Want to avoid AI generation costs + +**Use AI Generation When:** +- Need unique/different result each time +- Different angle or perspective needed +- Want variation and creativity +- Patch doesn't fit the context + +--- + +## Pro Tips + +### Building a Good Patch Library + +1. **Save your best AI results** - When AI generates something great, save it immediately +2. **Organize with categories** - Use consistent categories for easy finding +3. **Tag descriptively** - Include orientation (left/right), pose, lighting, etc. +4. **Create variations** - Save multiple versions of common needs (left hand, right hand, etc.) +5. **Build gradually** - Your library becomes more valuable over time + +### Maximizing Quality + +1. **For hands:** Always use `realistic-vision` model or save good results +2. **For faces:** Use Mode B (full image context) for better matching +3. **Use high feather values** (15-20px) when applying saved patches +4. **Test positioning** before finalizing - patches work best when lighting/angle matches + +### Saving Money + +1. **Build a patch library** of common needs +2. **Use `lama` for removals** instead of expensive models +3. **Let auto-selection work** - it picks the cheapest appropriate model +4. **Reuse successful patches** instead of regenerating + +--- + +## API Quick Reference + +```bash +# List available patches +GET /patches/ + +# Get patch details +GET /patches/{id} + +# Get patch image +GET /patches/{id}/image +GET /patches/{id}/image?thumbnail=true + +# Create patch from AI edit +POST /patches/ +{ + "name": "My Patch", + "source_type": "ai_generated", + "source_edit_id": 123, + "category": "hand" +} + +# Create patch from manual selection +POST /patches/ +{ + "name": "My Patch", + "source_type": "manual_selection", + "source_project_id": 456, + "bbox": {"x": 100, "y": 100, "width": 200, "height": 200} +} + +# Apply saved patch +POST /patches/apply +{ + "project_id": 789, + "patch_id": 123, + "bbox": {"x": 300, "y": 400, "width": 200, "height": 200}, + "feather_px": 10 +} + +# Delete patch +DELETE /patches/{id} + +# Update patch metadata +PUT /patches/{id} +{ + "name": "Updated Name", + "tags": "new, tags", + "category": "hand" +} +``` + +--- + +## Summary + +✅ **For hands/faces/bodies:** Use `realistic-vision` model +✅ **For removal:** Use `lama` model +✅ **For general edits:** Use `sdxl-inpaint` (default) +✅ **Auto-selection works great** - just write natural prompts +✅ **Save good AI results** to patch library for reuse +✅ **Save manual selections** from any image +✅ **Reuse patches across images** to save money and maintain consistency + +**You now have the best of both worlds:** +- AI generation when you need something new +- Saved patches when you need consistency or want to save money