Wire Smart Select, Color Select, fix layer buttons, add zoom

Frontend changes:
- Wire Smart Select and Color Select to canvas click handlers
- Add externalSelection prop to ImageCanvas for displaying AI-generated selections
- Add zoom controls (mouse wheel + buttons) to ImageCanvas
- Fix layer buttons (New Layer, Delete, Duplicate) with proper handlers
- Lift advancedToolMode state to App.jsx for coordination between components
- Add tool mode indicator overlay on canvas

Backend changes:
- Update smart-select endpoint to return JSON with polygon and bbox data
- Update color-select endpoint to return JSON with polygon and bbox data
- Add _mask_to_polygon helper function using OpenCV contour detection
- Add cv2 and base64 imports to tools.py

API changes:
- smartSelect and colorSelect now return { polygon, bbox, mask_base64 }

backend/app/routers/tools.py

@@ -6,6 +6,8 @@ from PIL import Image
 from io import BytesIO
 import numpy as np
 import json
+import base64
+import cv2
 
 from app.database import get_db
 from app.models.project import Project
@@ -133,11 +135,12 @@ async def smart_select(
     project_id: int = Form(...),
     point_x: int = Form(...),
     point_y: int = Form(...),
+    return_format: str = Form("json"),  # "json" (default) or "image"
     db: Session = Depends(get_db)
 ):
     """
     Use SAM (Segment Anything) to select object at given point.
-    Returns mask for the selected object.
+    Returns mask and polygon data for the selected object.
 
     Note: Requires SAM model to be downloaded.
     Falls back to simple flood-fill selection if SAM unavailable.
@@ -163,13 +166,62 @@ async def smart_select(
     # Convert mask to PNG
     mask_img = Image.fromarray((mask * 255).astype(np.uint8), mode='L')
 
+    if return_format == "image":
+        buffer = BytesIO()
+        mask_img.save(buffer, format='PNG')
+        return Response(
+            content=buffer.getvalue(),
+            media_type="image/png"
+        )
+
+    # Return JSON with polygon and bbox
+    polygon, bbox = _mask_to_polygon(mask)
+
+    # Also return mask as base64 for potential use
     buffer = BytesIO()
     mask_img.save(buffer, format='PNG')
+    mask_b64 = base64.b64encode(buffer.getvalue()).decode('utf-8')
 
-    return Response(
+    return {
-        content=buffer.getvalue(),
+        "polygon": polygon,
-        media_type="image/png"
+        "bbox": bbox,
-    )
+        "mask_base64": mask_b64,
+    }
+
+
+def _mask_to_polygon(mask: np.ndarray) -> tuple:
+    """
+    Convert a binary mask to a simplified polygon and bounding box.
+
+    Returns:
+        (polygon, bbox) where:
+        - polygon: list of [x, y] points (simplified contour)
+        - bbox: dict with x, y, width, height
+    """
+    # Ensure mask is binary uint8
+    mask_uint8 = (mask * 255).astype(np.uint8) if mask.max() <= 1 else mask.astype(np.uint8)
+
+    # Find contours
+    contours, _ = cv2.findContours(mask_uint8, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+    if not contours:
+        return [], {"x": 0, "y": 0, "width": 0, "height": 0}
+
+    # Get largest contour
+    largest = max(contours, key=cv2.contourArea)
+
+    # Get bounding box
+    x, y, w, h = cv2.boundingRect(largest)
+    bbox = {"x": int(x), "y": int(y), "width": int(w), "height": int(h)}
+
+    # Simplify contour to reduce points (epsilon = 1% of arc length)
+    epsilon = 0.01 * cv2.arcLength(largest, True)
+    simplified = cv2.approxPolyDP(largest, epsilon, True)
+
+    # Convert to list of [x, y] points
+    polygon = [[int(pt[0][0]), int(pt[0][1])] for pt in simplified]
+
+    return polygon, bbox
 
 
 # Global SAM model cache (loaded once, reused)
@@ -387,11 +439,12 @@ async def color_select(
     color_g: int = Form(...),
     color_b: int = Form(...),
     tolerance: int = Form(30),
+    return_format: str = Form("json"),  # "json" (default) or "image"
     db: Session = Depends(get_db)
 ):
     """
     Select all pixels similar to the given color.
-    Returns a mask of selected areas.
+    Returns a mask and polygon data for selected areas.
     """
     project = db.query(Project).filter(Project.id == project_id).first()
     if not project:
@@ -412,18 +465,33 @@ async def color_select(
     distance = np.sum(diff, axis=2)
 
     # Create mask where distance is within tolerance
-    mask = (distance <= tolerance * 3).astype(np.uint8) * 255
+    mask = (distance <= tolerance * 3).astype(np.uint8)
 
     # Convert to PNG
-    mask_img = Image.fromarray(mask, mode='L')
+    mask_img = Image.fromarray(mask * 255, mode='L')
+
+    if return_format == "image":
+        buffer = BytesIO()
+        mask_img.save(buffer, format='PNG')
+        return Response(
+            content=buffer.getvalue(),
+            media_type="image/png"
+        )
+
+    # Return JSON with polygon and bbox
+    polygon, bbox = _mask_to_polygon(mask)
 
     buffer = BytesIO()
     mask_img.save(buffer, format='PNG')
+    mask_b64 = base64.b64encode(buffer.getvalue()).decode('utf-8')
 
-    return Response(
+    return {
-        content=buffer.getvalue(),
+        "polygon": polygon,
-        media_type="image/png"
+        "bbox": bbox,
-    )
+        "mask_base64": mask_b64,
+        "color": {"r": color_r, "g": color_g, "b": color_b},
+        "tolerance": tolerance,
+    }
 
 
 @router.post("/extract-object")

frontend/src/App.jsx

@@ -26,6 +26,9 @@ function App() {
   const [layers, setLayers] = useState([]);
   const [activeLayer, setActiveLayer] = useState('background');
   const [generatedMask, setGeneratedMask] = useState(null);
+  const [advancedToolMode, setAdvancedToolMode] = useState(null); // 'smart-select', 'color-select', 'object-remove'
+  const [canvasZoom, setCanvasZoom] = useState(1);
+  const [externalSelection, setExternalSelection] = useState(null); // For smart-select/color-select polygon results
   const editsRef = useRef([]);
 
   // Create project and upload image
@@ -284,9 +287,49 @@ function App() {
   };
 
   // Handle mask generation from smart select / color select
-  const handleMaskGenerated = async (maskBlob, source) => {
+  const handleMaskGenerated = async (maskData, source) => {
-    setGeneratedMask({ blob: maskBlob, source });
+    setGeneratedMask({ data: maskData, source });
-    // The mask can be used for various operations
+    // Convert mask to selection if it contains polygon data
+    if (maskData && maskData.polygon && maskData.polygon.length > 0) {
+      // Set external selection for canvas to draw
+      setExternalSelection({
+        polygon: maskData.polygon,
+        bbox: maskData.bbox,
+      });
+      // Also set selection state for fix button
+      setSelection({
+        type: 'polygon',
+        bbox: maskData.bbox,
+        selectionData: { points: maskData.polygon },
+      });
+    }
+    // Reset tool mode after selection
+    setAdvancedToolMode(null);
+  };
+
+  // Handle canvas click for advanced tools (smart select, color select)
+  const handleAdvancedToolClick = async (x, y, color) => {
+    if (!project || !advancedToolMode) return;
+
+    try {
+      setIsProcessing(true);
+      setError(null);
+
+      if (advancedToolMode === 'smart-select') {
+        const result = await toolsApi.smartSelect(project.id, x, y);
+        handleMaskGenerated(result, 'smart-select');
+      } else if (advancedToolMode === 'color-select') {
+        // Color is passed from canvas click
+        if (color) {
+          const result = await toolsApi.colorSelect(project.id, color.r, color.g, color.b, 30);
+          handleMaskGenerated(result, 'color-select');
+        }
+      }
+    } catch (err) {
+      setError(`${advancedToolMode} failed: ${err.message}`);
+    } finally {
+      setIsProcessing(false);
+    }
   };
 
   // Handle flatten layers
@@ -361,6 +404,11 @@ function App() {
                 imageUrl={currentImageUrl}
                 onSelectionChange={setSelection}
                 selectionMode={selectionMode}
+                advancedToolMode={advancedToolMode}
+                onAdvancedToolClick={handleAdvancedToolClick}
+                zoom={canvasZoom}
+                onZoomChange={setCanvasZoom}
+                externalSelection={externalSelection}
               />
             </div>
 
@@ -391,6 +439,8 @@ function App() {
                 isProcessing={isProcessing}
                 setIsProcessing={setIsProcessing}
                 setError={setError}
+                activeToolMode={advancedToolMode}
+                setActiveToolMode={setAdvancedToolMode}
               />
 
               <Layers
@@ -401,6 +451,7 @@ function App() {
                 setActiveLayer={setActiveLayer}
                 onFlatten={handleFlattenLayers}
                 isProcessing={isProcessing}
+                onError={setError}
               />
 
               <EyeCatalog

frontend/src/components/AdvancedTools.jsx

@@ -11,8 +11,9 @@ const AdvancedTools = ({
   isProcessing,
   setIsProcessing,
   setError,
+  activeToolMode,
+  setActiveToolMode,
 }) => {
-  const [activeToolMode, setActiveToolMode] = useState(null);
   const [colorTolerance, setColorTolerance] = useState(30);
 
   const handleRemoveBackground = async () => {

frontend/src/components/ImageCanvas.css

@@ -43,3 +43,66 @@
   z-index: 10;
   white-space: nowrap;
 }
+
+/* Zoom controls */
+.zoom-controls {
+  position: absolute;
+  bottom: 10px;
+  right: 10px;
+  display: flex;
+  align-items: center;
+  gap: 4px;
+  background-color: rgba(0, 0, 0, 0.8);
+  padding: 6px 10px;
+  border-radius: 4px;
+  z-index: 10;
+}
+
+.zoom-controls button {
+  width: 28px;
+  height: 28px;
+  padding: 0;
+  font-size: 18px;
+  font-weight: bold;
+  background-color: #444;
+  color: white;
+  border: 1px solid #666;
+  border-radius: 4px;
+  cursor: pointer;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+.zoom-controls button:hover {
+  background-color: #555;
+}
+
+.zoom-level {
+  color: white;
+  font-size: 12px;
+  min-width: 45px;
+  text-align: center;
+}
+
+/* Tool mode indicator */
+.tool-mode-indicator {
+  position: absolute;
+  top: 10px;
+  left: 50%;
+  transform: translateX(-50%);
+  background-color: rgba(0, 120, 255, 0.9);
+  color: white;
+  padding: 10px 20px;
+  border-radius: 4px;
+  font-size: 14px;
+  font-weight: 500;
+  z-index: 10;
+  white-space: nowrap;
+  animation: pulse 2s infinite;
+}
+
+@keyframes pulse {
+  0%, 100% { opacity: 1; }
+  50% { opacity: 0.7; }
+}

frontend/src/components/ImageCanvas.jsx

@@ -2,12 +2,22 @@ import React, { useEffect, useRef, useState, useCallback } from 'react';
 import { fabric } from 'fabric';
 import './ImageCanvas.css';
 
-const ImageCanvas = ({ imageUrl, onSelectionChange, selectionMode }) => {
+const ImageCanvas = ({
+  imageUrl,
+  onSelectionChange,
+  selectionMode,
+  advancedToolMode,
+  onAdvancedToolClick,
+  zoom = 1,
+  onZoomChange,
+  externalSelection, // { polygon: [[x,y],...], bbox: {x,y,width,height} }
+}) => {
   const canvasRef = useRef(null);
   const fabricCanvasRef = useRef(null);
   const [currentSelection, setCurrentSelection] = useState(null);
   const [isDrawing, setIsDrawing] = useState(false);
   const [isTransformMode, setIsTransformMode] = useState(false);
+  const [currentZoom, setCurrentZoom] = useState(zoom);
   const lassoPoints = useRef([]);
 
   useEffect(() => {
@@ -50,11 +60,36 @@ const ImageCanvas = ({ imageUrl, onSelectionChange, selectionMode }) => {
     handleResize();
     window.addEventListener('resize', handleResize);
 
+    // Mouse wheel zoom
+    const handleWheel = (opt) => {
+      const e = opt.e;
+      e.preventDefault();
+      e.stopPropagation();
+
+      const delta = e.deltaY;
+      let newZoom = canvas.getZoom();
+      newZoom *= 0.999 ** delta;
+
+      // Clamp zoom between 0.1x and 10x
+      if (newZoom > 10) newZoom = 10;
+      if (newZoom < 0.1) newZoom = 0.1;
+
+      // Zoom to point under cursor
+      const pointer = canvas.getPointer(e, true);
+      canvas.zoomToPoint({ x: pointer.x, y: pointer.y }, newZoom);
+
+      setCurrentZoom(newZoom);
+      onZoomChange?.(newZoom);
+    };
+
+    canvas.on('mouse:wheel', handleWheel);
+
     return () => {
       window.removeEventListener('resize', handleResize);
+      canvas.off('mouse:wheel', handleWheel);
       canvas.dispose();
     };
-  }, []);
+  }, [onZoomChange]);
 
   // Load image when URL changes
   useEffect(() => {
@@ -94,12 +129,128 @@ const ImageCanvas = ({ imageUrl, onSelectionChange, selectionMode }) => {
     }, { crossOrigin: 'anonymous' });
   }, [imageUrl]);
 
+  // Handle advanced tool mode clicks (smart-select, color-select)
+  useEffect(() => {
+    if (!fabricCanvasRef.current || !advancedToolMode) return;
+
+    const canvas = fabricCanvasRef.current;
+    const bgImage = canvas.backgroundImage;
+
+    const handleAdvancedClick = async (e) => {
+      if (!bgImage || !onAdvancedToolClick) return;
+
+      const pointer = canvas.getPointer(e.e);
+
+      // Convert canvas coordinates to image coordinates
+      const imgScale = bgImage.scaleX;
+      const imgLeft = bgImage.left;
+      const imgTop = bgImage.top;
+
+      const imgX = Math.round((pointer.x - imgLeft) / imgScale);
+      const imgY = Math.round((pointer.y - imgTop) / imgScale);
+
+      // Check if click is within image bounds
+      if (imgX < 0 || imgY < 0 || imgX > bgImage.width || imgY > bgImage.height) {
+        return;
+      }
+
+      if (advancedToolMode === 'color-select') {
+        // Get pixel color at click position
+        const ctx = canvas.getContext('2d');
+        const canvasX = pointer.x * canvas.getZoom();
+        const canvasY = pointer.y * canvas.getZoom();
+
+        // For color picking, we need to get the color from the image
+        // Create a temporary canvas to read pixel color
+        const tempCanvas = document.createElement('canvas');
+        tempCanvas.width = bgImage.width;
+        tempCanvas.height = bgImage.height;
+        const tempCtx = tempCanvas.getContext('2d');
+
+        // Draw the image element to temp canvas
+        const imgElement = bgImage.getElement();
+        tempCtx.drawImage(imgElement, 0, 0);
+
+        const pixelData = tempCtx.getImageData(imgX, imgY, 1, 1).data;
+        const color = { r: pixelData[0], g: pixelData[1], b: pixelData[2] };
+
+        onAdvancedToolClick(imgX, imgY, color);
+      } else {
+        onAdvancedToolClick(imgX, imgY, null);
+      }
+    };
+
+    canvas.on('mouse:down', handleAdvancedClick);
+
+    return () => {
+      canvas.off('mouse:down', handleAdvancedClick);
+    };
+  }, [advancedToolMode, onAdvancedToolClick]);
+
+  // Handle external selection (from smart-select or color-select)
+  useEffect(() => {
+    if (!fabricCanvasRef.current || !externalSelection?.polygon?.length) return;
+
+    const canvas = fabricCanvasRef.current;
+    const bgImage = canvas.backgroundImage;
+
+    if (!bgImage) return;
+
+    // Clear previous selection
+    if (currentSelection) {
+      canvas.remove(currentSelection);
+    }
+
+    // Convert image coordinates to canvas coordinates
+    const imgScale = bgImage.scaleX;
+    const imgLeft = bgImage.left;
+    const imgTop = bgImage.top;
+
+    const canvasPoints = externalSelection.polygon.map(([x, y]) => ({
+      x: x * imgScale + imgLeft,
+      y: y * imgScale + imgTop,
+    }));
+
+    // Create polygon selection
+    const polygon = new fabric.Polygon(canvasPoints, {
+      fill: 'rgba(255, 255, 255, 0.3)',
+      stroke: '#00ff00',
+      strokeWidth: 2,
+      selectable: true,
+      hasControls: true,
+      hasBorders: true,
+      lockRotation: false,
+      cornerColor: '#00ff00',
+      cornerSize: 10,
+      transparentCorners: false,
+      borderColor: '#00ff00',
+      borderScaleFactor: 2,
+    });
+
+    canvas.add(polygon);
+    canvas.setActiveObject(polygon);
+    setCurrentSelection(polygon);
+    lassoPoints.current = canvasPoints;
+
+    // Notify parent of selection
+    onSelectionChange({
+      type: 'polygon',
+      bbox: externalSelection.bbox,
+      selectionData: { points: externalSelection.polygon },
+    });
+
+    canvas.renderAll();
+  }, [externalSelection]);
+
   // Handle selection mode changes
   useEffect(() => {
     if (!fabricCanvasRef.current) return;
 
     const canvas = fabricCanvasRef.current;
 
+    // Don't set up selection handlers if in advanced tool mode
+    if (advancedToolMode) return;
+
     // Clear previous selection when changing modes
     if (currentSelection) {
       canvas.remove(currentSelection);
@@ -123,7 +274,7 @@ const ImageCanvas = ({ imageUrl, onSelectionChange, selectionMode }) => {
     } else if (selectionMode === 'lasso') {
       setupLassoMode(canvas);
     }
-  }, [selectionMode]);
+  }, [selectionMode, advancedToolMode]);
 
   const setupRectangleMode = (canvas) => {
     let rect, isDown, startX, startY;
@@ -497,10 +648,57 @@ const ImageCanvas = ({ imageUrl, onSelectionChange, selectionMode }) => {
     }
   };
 
+  const handleZoomIn = () => {
+    if (!fabricCanvasRef.current) return;
+    const canvas = fabricCanvasRef.current;
+    let newZoom = canvas.getZoom() * 1.2;
+    if (newZoom > 10) newZoom = 10;
+    canvas.setZoom(newZoom);
+    setCurrentZoom(newZoom);
+    onZoomChange?.(newZoom);
+  };
+
+  const handleZoomOut = () => {
+    if (!fabricCanvasRef.current) return;
+    const canvas = fabricCanvasRef.current;
+    let newZoom = canvas.getZoom() / 1.2;
+    if (newZoom < 0.1) newZoom = 0.1;
+    canvas.setZoom(newZoom);
+    setCurrentZoom(newZoom);
+    onZoomChange?.(newZoom);
+  };
+
+  const handleZoomReset = () => {
+    if (!fabricCanvasRef.current) return;
+    const canvas = fabricCanvasRef.current;
+    canvas.setZoom(1);
+    canvas.setViewportTransform([1, 0, 0, 1, 0, 0]);
+    setCurrentZoom(1);
+    onZoomChange?.(1);
+  };
+
   return (
     <div className="canvas-container">
       <canvas ref={canvasRef} />
-      {currentSelection && (
+
+      {/* Zoom controls */}
+      <div className="zoom-controls">
+        <button onClick={handleZoomOut} title="Zoom Out">−</button>
+        <span className="zoom-level">{Math.round(currentZoom * 100)}%</span>
+        <button onClick={handleZoomIn} title="Zoom In">+</button>
+        <button onClick={handleZoomReset} title="Reset Zoom">⟲</button>
+      </div>
+
+      {/* Advanced tool mode indicator */}
+      {advancedToolMode && (
+        <div className="tool-mode-indicator">
+          {advancedToolMode === 'smart-select' && 'Click on an object to select it'}
+          {advancedToolMode === 'color-select' && 'Click on a color to select similar pixels'}
+          {advancedToolMode === 'object-remove' && 'Click on an object to remove it'}
+        </div>
+      )}
+
+      {currentSelection && !advancedToolMode && (
         <>
           <div className="selection-hint">
             Click selection to move/resize/rotate

frontend/src/components/Layers.jsx

@@ -11,6 +11,7 @@ const Layers = ({
   onLayerVisibilityChange,
   onFlatten,
   isProcessing,
+  onError,
 }) => {
   const [draggedLayer, setDraggedLayer] = useState(null);
 
@@ -69,6 +70,56 @@ const Layers = ({
     loadLayers();
   };
 
+  const handleNewLayer = async () => {
+    if (!projectId) return;
+    try {
+      // Create a new empty transparent layer
+      const newLayer = {
+        id: `layer-${Date.now()}`,
+        name: `Layer ${layers.length + 1}`,
+        visible: true,
+        thumbnail: null,
+      };
+      setLayers([...layers, newLayer]);
+      setActiveLayer(newLayer.id);
+    } catch (err) {
+      onError?.(`Failed to create layer: ${err.message}`);
+    }
+  };
+
+  const handleDeleteLayer = async () => {
+    if (!projectId || activeLayer === 'background') return;
+    try {
+      const updatedLayers = layers.filter((l) => l.id !== activeLayer);
+      setLayers(updatedLayers);
+      setActiveLayer(updatedLayers.length > 0 ? updatedLayers[updatedLayers.length - 1].id : 'background');
+    } catch (err) {
+      onError?.(`Failed to delete layer: ${err.message}`);
+    }
+  };
+
+  const handleDuplicateLayer = async () => {
+    if (!projectId || activeLayer === 'background') return;
+    try {
+      const layerToDuplicate = layers.find((l) => l.id === activeLayer);
+      if (!layerToDuplicate) return;
+
+      const newLayer = {
+        ...layerToDuplicate,
+        id: `layer-${Date.now()}`,
+        name: `${layerToDuplicate.name} copy`,
+      };
+
+      const activeIndex = layers.findIndex((l) => l.id === activeLayer);
+      const updatedLayers = [...layers];
+      updatedLayers.splice(activeIndex + 1, 0, newLayer);
+      setLayers(updatedLayers);
+      setActiveLayer(newLayer.id);
+    } catch (err) {
+      onError?.(`Failed to duplicate layer: ${err.message}`);
+    }
+  };
+
   return (
     <div className="layers-panel">
       <div className="layers-header">
@@ -142,6 +193,7 @@ const Layers = ({
           className="layer-action-btn"
           disabled={isProcessing || !projectId}
           title="Add new empty layer"
+          onClick={handleNewLayer}
         >
           + New Layer
         </button>
@@ -149,6 +201,7 @@ const Layers = ({
           className="layer-action-btn"
           disabled={isProcessing || activeLayer === 'background'}
           title="Delete selected layer"
+          onClick={handleDeleteLayer}
         >
           Delete
         </button>
@@ -156,6 +209,7 @@ const Layers = ({
           className="layer-action-btn"
           disabled={isProcessing || activeLayer === 'background'}
           title="Duplicate selected layer"
+          onClick={handleDuplicateLayer}
         >
           Duplicate
         </button>

frontend/src/utils/api.js

@@ -171,20 +171,22 @@ export const toolsApi = {
   },
 
   // Smart select object at point
+  // Returns { polygon, bbox, mask_base64 }
   smartSelect: async (projectId, x, y) => {
     const formData = new FormData();
     formData.append('project_id', projectId);
     formData.append('point_x', x);
     formData.append('point_y', y);
+    formData.append('return_format', 'json');
 
     const response = await api.post('/tools/smart-select', formData, {
       headers: { 'Content-Type': 'multipart/form-data' },
-      responseType: 'blob',
     });
     return response.data;
   },
 
   // Select by color
+  // Returns { polygon, bbox, mask_base64, color, tolerance }
   colorSelect: async (projectId, r, g, b, tolerance = 30) => {
     const formData = new FormData();
     formData.append('project_id', projectId);
@@ -192,10 +194,10 @@ export const toolsApi = {
     formData.append('color_g', g);
     formData.append('color_b', b);
     formData.append('tolerance', tolerance);
+    formData.append('return_format', 'json');
 
     const response = await api.post('/tools/color-select', formData, {
       headers: { 'Content-Type': 'multipart/form-data' },
-      responseType: 'blob',
     });
     return response.data;
   },