Warning: Permanently added 'carl' (ED25519) to the list of known hosts.
/opt/ghc20/backend/app/services/blog_generator.py:                - image_url: URL to generated collage image
/opt/ghc20/backend/app/services/blog_generator.py-        """
/opt/ghc20/backend/app/services/blog_generator.py-        if not deals:
/opt/ghc20/backend/app/services/blog_generator.py-            logger.warning("No deals provided for blog generation")
/opt/ghc20/backend/app/services/blog_generator.py-            return self._generate_fallback_content(category, time_period)
/opt/ghc20/backend/app/services/blog_generator.py-
/opt/ghc20/backend/app/services/blog_generator.py-        try:
/opt/ghc20/backend/app/services/blog_generator.py-            # Build prompt for AI
/opt/ghc20/backend/app/services/blog_generator.py-            prompt = self._build_roundup_prompt(category, deals, time_period)
/opt/ghc20/backend/app/services/blog_generator.py-
/opt/ghc20/backend/app/services/blog_generator.py-            # Generate content using AI - prioritize Gemini 2.5 Flash Lite
/opt/ghc20/backend/app/services/blog_generator.py-            if self.use_gemini:
/opt/ghc20/backend/app/services/blog_generator.py-                content = await self._generate_with_gemini(prompt)
/opt/ghc20/backend/app/services/blog_generator.py-            elif self.use_anthropic:
/opt/ghc20/backend/app/services/blog_generator.py-                content = await self._generate_with_anthropic(prompt)
/opt/ghc20/backend/app/services/blog_generator.py-            else:
/opt/ghc20/backend/app/services/blog_generator.py-                content = await self._generate_with_local_llm(prompt)
/opt/ghc20/backend/app/services/blog_generator.py-
/opt/ghc20/backend/app/services/blog_generator.py-            # Parse AI response into structured data
/opt/ghc20/backend/app/services/blog_generator.py-            result = self._parse_ai_response(content, category, time_period)
/opt/ghc20/backend/app/services/blog_generator.py-
/opt/ghc20/backend/app/services/blog_generator.py-            # Fix HTML issues (LLM often makes mistakes)
/opt/ghc20/backend/app/services/blog_generator.py-            if result.get('content'):
/opt/ghc20/backend/app/services/blog_generator.py-                # Fix orphaned images (must be wrapped in <p> tags)
/opt/ghc20/backend/app/services/blog_generator.py-                result['content'] = self._fix_orphaned_images(result['content'])
/opt/ghc20/backend/app/services/blog_generator.py-                # Fix incorrect image URLs (match to correct products)
/opt/ghc20/backend/app/services/blog_generator.py:                result['content'] = self._fix_image_urls(result['content'], deals)
/opt/ghc20/backend/app/services/blog_generator.py-                # Inject internal backlinks for SEO
/opt/ghc20/backend/app/services/blog_generator.py-                result['content'] = self._inject_internal_backlinks(result['content'], deals)
/opt/ghc20/backend/app/services/blog_generator.py-
/opt/ghc20/backend/app/services/blog_generator.py-                # Humanize the content to make it read more naturally
/opt/ghc20/backend/app/services/blog_generator.py-                if self.use_gemini:
/opt/ghc20/backend/app/services/blog_generator.py-                    logger.info("Running humanizer pass on generated content...")
/opt/ghc20/backend/app/services/blog_generator.py-                    result['content'] = await self._humanize_content(result['content'])
/opt/ghc20/backend/app/services/blog_generator.py-
/opt/ghc20/backend/app/services/blog_generator.py-            # Generate collage from deal images
/opt/ghc20/backend/app/services/blog_generator.py:            image_urls = [deal.get('image_url') for deal in deals if deal.get('image_url')]
/opt/ghc20/backend/app/services/blog_generator.py:            if image_urls:
/opt/ghc20/backend/app/services/blog_generator.py-                try:
/opt/ghc20/backend/app/services/blog_generator.py-                    overlay_text = f"{time_period.title()} Roundup"
/opt/ghc20/backend/app/services/blog_generator.py-                    if category:
/opt/ghc20/backend/app/services/blog_generator.py-                        overlay_text = f"{category} {overlay_text}"
/opt/ghc20/backend/app/services/blog_generator.py-
/opt/ghc20/backend/app/services/blog_generator.py-                    # Generate unique filename
/opt/ghc20/backend/app/services/blog_generator.py-                    from datetime import datetime
/opt/ghc20/backend/app/services/blog_generator.py-                    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
/opt/ghc20/backend/app/services/blog_generator.py-                    filename = f"blog_collage_{timestamp}"
/opt/ghc20/backend/app/services/blog_generator.py-
/opt/ghc20/backend/app/services/blog_generator.py-                    collage_path = await self.collage_generator.generate_from_urls(
/opt/ghc20/backend/app/services/blog_generator.py:                        image_urls[:6],  # Use up to 6 images
/opt/ghc20/backend/app/services/blog_generator.py-                        filename,
/opt/ghc20/backend/app/services/blog_generator.py-                        overlay_text=overlay_text
/opt/ghc20/backend/app/services/blog_generator.py-                    )
/opt/ghc20/backend/app/services/blog_generator.py-
/opt/ghc20/backend/app/services/blog_generator.py-                    if collage_path:
/opt/ghc20/backend/app/services/blog_generator.py-                        # Convert file path to URL path for frontend
/opt/ghc20/backend/app/services/blog_generator.py-                        from pathlib import Path
/opt/ghc20/backend/app/services/blog_generator.py-                        import shutil
/opt/ghc20/backend/app/services/blog_generator.py-                        filename_only = Path(collage_path).name
/opt/ghc20/backend/app/services/blog_generator.py:                        result['image_url'] = f"/blog-images/{filename_only}"
/opt/ghc20/backend/app/services/blog_generator.py:                        logger.info(f"Generated blog collage: {collage_path} -> {result['image_url']}")
/opt/ghc20/backend/app/services/blog_generator.py-
/opt/ghc20/backend/app/services/blog_generator.py-                        # Also copy to NPM data directory for nginx serving (production)
/opt/ghc20/backend/app/services/blog_generator.py-                        npm_blog_images = Path("/home/crogers2287/nginx-proxy-manager/data/blog-images")
/opt/ghc20/backend/app/services/blog_generator.py-                        if npm_blog_images.exists():
/opt/ghc20/backend/app/services/blog_generator.py-                            try:
/opt/ghc20/backend/app/services/blog_generator.py-                                shutil.copy2(collage_path, npm_blog_images / filename_only)
/opt/ghc20/backend/app/services/blog_generator.py-                                logger.info(f"Copied collage to NPM directory: {npm_blog_images / filename_only}")
/opt/ghc20/backend/app/services/blog_generator.py-                            except Exception as copy_err:
/opt/ghc20/backend/app/services/blog_generator.py-                                logger.warning(f"Failed to copy to NPM directory: {copy_err}")
/opt/ghc20/backend/app/services/blog_generator.py-                except Exception as e:
/opt/ghc20/backend/app/services/blog_generator.py-                    logger.warning(f"Failed to generate collage: {e}")
/opt/ghc20/backend/app/services/blog_generator.py-
/opt/ghc20/backend/app/services/blog_generator.py-            return result
/opt/ghc20/backend/app/services/blog_generator.py-
/opt/ghc20/backend/app/services/blog_generator.py-        except Exception as e:
/opt/ghc20/backend/app/services/blog_generator.py-            logger.error(f"Failed to generate blog post: {e}")
/opt/ghc20/backend/app/services/blog_generator.py-            return self._generate_fallback_content(category, time_period)
/opt/ghc20/backend/app/services/blog_generator.py-
/opt/ghc20/backend/app/services/blog_generator.py-    def _build_roundup_prompt(


Let me find the actual scraper code that downloads and saves deal images. I need to understand how images get from the source URL to the stored file.

exec

# Find the scraper that downloads deal images
ssh carl "find /opt/ghc20 -name '*.py' -exec grep -l 'download.*image\|save.*image\|image.*hash\|deal_images' {} \; 2>/dev/null"

# Check how image URLs are generated in the deals
ssh carl "grep -r 'image_url' /opt/ghc20/backend/app/services/*.py 2>/dev/null | grep -v '.pyc' | head -20"


Command still running (session fast-nexus, pid 1603601). Use process (list/poll/log/write/kill/clear/remove) for follow-up.

Warning: Permanently added 'carl' (ED25519) to the list of known hosts.
/opt/ghc20/novel-os/venv/lib/python3.12/site-packages/pip/_vendor/pygments/formatters/img.py
/opt/ghc20/hash_based_recovery.py
/opt/ghc20/venv/lib/python3.11/site-packages/pygments/formatters/img.py


Warning: Permanently added 'carl' (ED25519) to the list of known hosts.
/opt/ghc20/backend/app/services/blog_generator.py:                - image_url: URL to generated collage image
/opt/ghc20/backend/app/services/blog_generator.py:                result['content'] = self._fix_image_urls(result['content'], deals)
/opt/ghc20/backend/app/services/blog_generator.py:            image_urls = [deal.get('image_url') for deal in deals if deal.get('image_url')]
/opt/ghc20/backend/app/services/blog_generator.py:            if image_urls:
/opt/ghc20/backend/app/services/blog_generator.py:                        image_urls[:6],  # Use up to 6 images
/opt/ghc20/backend/app/services/blog_generator.py:                        result['image_url'] = f"/blog-images/{filename_only}"
/opt/ghc20/backend/app/services/blog_generator.py:                        logger.info(f"Generated blog collage: {collage_path} -> {result['image_url']}")
/opt/ghc20/backend/app/services/blog_generator.py:            image_url = deal.get('image_url', '')
/opt/ghc20/backend/app/services/blog_generator.py:            if image_url:
/opt/ghc20/backend/app/services/blog_generator.py:                deal_text += "\n  IMAGE URL (use this exact URL for THIS deal only): " + str(image_url)
/opt/ghc20/backend/app/services/blog_generator.py:    def _fix_image_urls(self, html_content: str, deals: List[Dict]) -> str:
/opt/ghc20/backend/app/services/blog_generator.py:            image_url = deal.get('image_url', '')
/opt/ghc20/backend/app/services/blog_generator.py:            if deal_url and image_url:
/opt/ghc20/backend/app/services/blog_generator.py:                url_to_image[affiliate_url] = image_url
/opt/ghc20/backend/app/services/daily_roundup_generator.py:async def load_image(image_url: str) -> Optional[Image.Image]:
/opt/ghc20/backend/app/services/daily_roundup_generator.py:        if image_url.startswith("/"):
/opt/ghc20/backend/app/services/daily_roundup_generator.py:            local_path = PUBLIC_DIR / image_url.lstrip("/")
/opt/ghc20/backend/app/services/daily_roundup_generator.py:                response = await client.get(image_url, headers=headers)
/opt/ghc20/backend/app/services/daily_roundup_generator.py:        logger.error(f"Failed to load image from {image_url}: {e}")
/opt/ghc20/backend/app/services/daily_roundup_generator.py:                .where(Deal.image_url != None)


Let me check the find command and also look at where images actually get stored/downloaded. The scraper code should be in the scrapers directory.

# Look at the deal model to understand image storage
ssh carl "cat /opt/ghc20/backend/app/models/deal.py 2>/dev/null | grep -A5 -B5 'image'"

# Check the deal-images directory
ssh carl "ls -la /opt/ghc20/frontend/public/deal-images/ 2>/dev/null | head -20 || ls -la /home/crogers2287/nginx-proxy-manager/data/deal-images/ 2>/dev/null | head -20"


Warning: Permanently added 'carl' (ED25519) to the list of known hosts.

    # Multi-retailer support (JSON array of {retailer, url, affiliate_url})
    retailer_links: Mapped[list | None] = mapped_column(JSON)

    # Images
    image_url: Mapped[str | None] = mapped_column(String(2000))
    thumbnail_url: Mapped[str | None] = mapped_column(String(2000))

    # Categorization
    category_id: Mapped[int | None] = mapped_column(ForeignKey("categories.id"), index=True)
    # Featured/promotional categories (e.g., ["black-friday-2025", "cyber-monday"])


Warning: Permanently added 'carl' (ED25519) to the list of known hosts.
total 113012
drwxr-xr-x 2 ghc20 ghc20  409600 Feb 19 03:15 .
drwxr-xr-x 6 ghc20 ghc20    4096 Feb 19 01:50 ..
-rw-r--r-- 1 ghc20 ghc20   21764 Feb 15 09:15 00374f8d8b91eb2a.jpg
-rw-r--r-- 1 ghc20 ghc20   27431 Feb 18 21:30 00470a0ce4709e42.jpg
-rw-r--r-- 1 ghc20 ghc20   29158 Feb 13 01:12 0052e9ee6c521950.jpg
-rw-r--r-- 1 ghc20 ghc20   35373 Feb 16 18:15 00563e76983b434b.jpg
-rw-r--r-- 1 ghc20 ghc20   17105 Dec  3 21:45 005b3cd6aa315a7a.jpg
-rw-r--r-- 1 ghc20 ghc20  130174 Feb 13 06:00 0060a48c2705a7a2.jpg
-rw-r--r-- 1 ghc20 ghc20   14065 Feb 17 20:45 0078544cfc687e51.jpg
-rw-r--r-- 1 ghc20 ghc20   17705 Feb 18 17:45 0096da27216827e6.jpg
-rw-r--r-- 1 ghc20 ghc20   18165 Feb 17 00:15 00a03890b20194b7.jpg
-rw-r--r-- 1 ghc20 ghc20   14944 Feb 18 06:15 00a44b0b44fe9d9d.jpg
-rw-r--r-- 1 ghc20 ghc20   37958 Feb 17 21:00 00a6e8b900b47033.jpg
-rw-r--r-- 1 ghc20 ghc20   21864 Feb 17 14:20 00a7c9a61447e660.jpg
-rw-r--r-- 1 ghc20 ghc20    5929 Feb 18 04:30 00afe753fc3259ac.jpg
-rw-r--r-- 1 ghc20 ghc20   14489 Feb 13 03:30 00c6c892592ecc95.jpg
-rw-r--r-- 1 ghc20 ghc20    5176 Feb 15 01:00 00d631314013961a.jpg
-rw-r--r-- 1 ghc20 ghc20   17699 Feb 17 08:30 00d743ab92fd1f73.jpg
-rw-r--r-- 1 ghc20 ghc20   19306 Feb 13 14:30 00ed63d800e55954.jpg


Found the image_downloader service! Let me check it - this is likely where the bug is.

ssh carl "cat /opt/ghc20/backend/app/services/image_downloader.py"


Warning: Permanently added 'carl' (ED25519) to the list of known hosts.
"""Image Downloader Service

Downloads images from external URLs and stores them locally to avoid broken CDN links.
Uses content-based hashing to avoid duplicates and handles errors gracefully.
"""
import asyncio
import hashlib
import httpx
from pathlib import Path
from typing import Optional
import logging
from datetime import datetime
import mimetypes

class ImageDownloader:
    """Service for downloading and storing images locally"""

    def __init__(
        self,
        storage_dir: Optional[str] = None,
        timeout: float = 10.0,
        max_size_mb: int = 5,
    ):
        """
        Initialize image downloader

        Args:
            storage_dir: Directory to store downloaded images (auto-detects sandbox vs production)
            timeout: HTTP request timeout in seconds
            max_size_mb: Maximum image size in megabytes
        """
        # Auto-detect environment: use /workspace in sandbox, absolute path in production
        if storage_dir is None:
            if Path('/workspace').exists():
                # Running in sandbox - use /workspace path
                storage_dir = "/workspace/frontend/public/deal-images"
            else:
                # Running in production - use absolute path
                storage_dir = "/opt/ghc20/frontend/public/deal-images"

        self.storage_dir = Path(storage_dir)
        self.timeout = timeout
        self.max_size_bytes = max_size_mb * 1024 * 1024
        self.logger = logging.getLogger(__name__)

        # Secondary storage for NPM (production only)
        self.npm_storage_dir = None
        npm_path = Path("/home/crogers2287/nginx-proxy-manager/data/deal-images")
        if npm_path.exists():
            self.npm_storage_dir = npm_path
            self.logger.info(f"NPM storage directory: {self.npm_storage_dir}")

        # Create storage directory if it doesn't exist
        self.storage_dir.mkdir(parents=True, exist_ok=True)
        self.logger.info(f"Image storage directory: {self.storage_dir}")

    def _get_content_hash(self, content: bytes) -> str:
        """Generate SHA-256 hash of image content"""
        return hashlib.sha256(content).hexdigest()[:16]  # Use first 16 chars

    def _get_file_extension(self, content_type: Optional[str], url: str) -> str:
        """
        Determine file extension from content type or URL

        Args:
            content_type: HTTP Content-Type header value
            url: Original image URL

        Returns:
            File extension (e.g., '.jpg', '.png')
        """
        # Try to get extension from content type
        if content_type:
            ext = mimetypes.guess_extension(content_type.split(';')[0].strip())
            if ext:
                # Normalize extensions
                if ext == '.jpe':
                    return '.jpg'
                if ext == '.jpeg':
                    return '.jpg'
                return ext

        # Fallback to URL extension
        url_ext = Path(url.split('?')[0]).suffix.lower()
        if url_ext in ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg']:
            if url_ext == '.jpeg':
                return '.jpg'
            return url_ext

        # Default to .jpg
        return '.jpg'

    def _is_valid_image_url(self, url: str) -> bool:
        """
        Check if URL looks like a valid image URL

        Args:
            url: URL to validate

        Returns:
            True if URL appears to be an image
        """
        if not url:
            return False

        url_lower = url.lower()

        # Check for common image extensions
        if any(ext in url_lower for ext in ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg']):
            return True

        # Check for common image CDNs
        image_domains = [
            'slickdealscdn.com',
            'cloudfront.net',
            'imgur.com',
            'amazonaws.com',
            'googleusercontent.com',
            'wp.com',
            'shopify.com',
        ]

        return any(domain in url_lower for domain in image_domains)

    async def download_and_store_image(self, url: str) -> str:
        """
        Download image from URL and store locally

        Args:
            url: External image URL to download

        Returns:
            Local URL path (e.g., '/deal-images/abc123.jpg') on success,
            original URL on failure
        """
        # Validate URL
        if not url or not url.startswith(('http://', 'https://')):
            self.logger.warning(f"Invalid URL format: {url}")
            return url

        # Quick validation check
        if not self._is_valid_image_url(url):
            self.logger.debug(f"URL doesn't appear to be an image: {url}")
            return url

        try:
            # Download image with timeout and size limits
            async with httpx.AsyncClient(timeout=self.timeout) as client:
                self.logger.debug(f"Downloading image: {url[:100]}...")

                response = await client.get(
                    url,
                    headers={
                        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
                    },
                    follow_redirects=True,
                )

                # Check response status
                if response.status_code != 200:
                    self.logger.warning(f"HTTP {response.status_code} for {url[:100]}")
                    return url

                # Check content type
                content_type = response.headers.get('Content-Type', '')
                if not content_type.startswith('image/'):
                    self.logger.warning(f"Non-image content type '{content_type}' for {url[:100]}")
                    return url

                # Get image content
                content = response.content

                # Check size
                if len(content) > self.max_size_bytes:
                    self.logger.warning(
                        f"Image too large ({len(content) / 1024 / 1024:.2f} MB) for {url[:100]}"
                    )
                    return url

                if len(content) < 100:  # Minimum viable image size
                    self.logger.warning(f"Image too small ({len(content)} bytes) for {url[:100]}")
                    return url

                # Generate content hash for deduplication
                content_hash = self._get_content_hash(content)

                # Determine file extension
                file_ext = self._get_file_extension(content_type, url)

                # Create filename
                filename = f"{content_hash}{file_ext}"
                file_path = self.storage_dir / filename

                # Check if file already exists (deduplication)
                if file_path.exists():
                    self.logger.debug(f"Image already exists: {filename}")
                    return f"/deal-images/{filename}"

                # Save image to disk
                file_path.write_bytes(content)

                # Also save to NPM storage directory if available (for production)
                if self.npm_storage_dir:
                    try:
                        npm_file_path = self.npm_storage_dir / filename
                        if not npm_file_path.exists():
                            npm_file_path.write_bytes(content)
                            self.logger.debug(f"Also saved to NPM storage: {filename}")
                    except Exception as npm_err:
                        self.logger.warning(f"Failed to save to NPM storage: {npm_err}")

                # Also save to built frontend directory for production serving
                try:
                    # Detect environment and use appropriate dist path
                    if Path("/opt/ghc20/frontend/dist").exists():
                        # Production environment
                        dist_base = "/opt/ghc20/frontend/dist"
                    elif Path("/workspace/frontend/dist").exists():
                        # Development sandbox
                        dist_base = "/workspace/frontend/dist"
                    else:
                        # Fallback to relative path
                        dist_base = str(Path(self.storage_dir).parent.parent / "dist")

                    dist_dir = Path(dist_base) / "deal-images"
                    dist_dir.mkdir(parents=True, exist_ok=True)
                    dist_file_path = dist_dir / filename
                    if not dist_file_path.exists():
                        dist_file_path.write_bytes(content)
                        self.logger.info(f"✅ Also saved to dist: {filename}")
                    else:
                        self.logger.debug(f"Image already exists in dist: {filename}")
                except Exception as dist_err:
                    self.logger.warning(f"Failed to save to dist directory: {dist_err}")

                self.logger.info(
                    f"✅ Downloaded and saved: {filename} "
                    f"({len(content) / 1024:.1f} KB) from {url[:80]}..."
                )

                return f"/deal-images/{filename}"

        except httpx.TimeoutException:
            self.logger.warning(f"Timeout downloading image: {url[:100]}")
            return url

        except httpx.HTTPError as e:
            self.logger.warning(f"HTTP error downloading image: {e} - {url[:100]}")
            return url

        except Exception as e:
            self.logger.error(f"Unexpected error downloading image: {e} - {url[:100]}")
            return url

    async def download_multiple_images(self, urls: list[str]) -> dict[str, str]:
        """
        Download multiple images concurrently

        Args:
            urls: List of image URLs to download

        Returns:
            Dictionary mapping original URL to local path
        """
        if not urls:
            return {}

        self.logger.info(f"Downloading {len(urls)} images...")

        # Download images concurrently
        tasks = [self.download_and_store_image(url) for url in urls]
        results = await asyncio.gather(*tasks)

        # Create mapping
        url_map = dict(zip(urls, results))

        # Count successes
        local_urls = [r for r in results if r.startswith('/deal-images/')]
        self.logger.info(
            f"Downloaded {len(local_urls)}/{len(urls)} images successfully"
        )

        return url_map

    def cleanup_old_images(self, days: int = 30) -> int:
        """
        Delete images older than specified days

        Args:
            days: Number of days to keep images

        Returns:
            Number of images deleted
        """
        if not self.storage_dir.exists():
            return 0

        from datetime import datetime, timedelta

        cutoff = datetime.now() - timedelta(days=days)
        deleted = 0

        for image_file in self.storage_dir.glob('*'):
            if image_file.is_file():
                # Check file modification time
                mtime = datetime.fromtimestamp(image_file.stat().st_mtime)
                if mtime < cutoff:
                    image_file.unlink()
                    deleted += 1

        self.logger.info(f"Cleaned up {deleted} images older than {days} days")
        return deleted

    def get_storage_stats(self) -> dict:
        """
        Get statistics about image storage

        Returns:
            Dictionary with storage statistics
        """
        if not self.storage_dir.exists():
            return {
                'total_images': 0,
                'total_size_mb': 0.0,
                'storage_dir': str(self.storage_dir),
            }

        files = list(self.storage_dir.glob('*'))
        total_size = sum(f.stat().st_size for f in files if f.is_file())

        return {
            'total_images': len([f for f in files if f.is_file()]),
            'total_size_mb': total_size / 1024 / 1024,
            'storage_dir': str(self.storage_dir),
        }


Found it! The image downloader uses **content hashing** - it generates a hash based on the actual image content bytes. This means:

1. If two different deals download the same image content, they'll get the same hash (good for deduplication)
2. BUT - if the scraper passes the WRONG image URL for a deal, it will download that wrong image and store it with a unique hash

The issue is NOT in the image downloader - it's in the **scraper** that provides the image URLs. Let me check the scraper code to see if there's a bug where deals are getting wrong image URLs assigned.

Actually, wait - let me also check if the issue could be with the posting script. Let me look at what happens when the posting script runs - specifically, does it fetch the deal fresh from the API each time?


#!/usr/bin/env python3
"""
GHC Deal Poster - End-to-end deal posting script
NO SHELL COMMANDS WITH $ - Pure Python to avoid variable expansion bugs

Usage:
  python3 post_deal_safe.py                    # Auto-select from feed
  python3 post_deal_safe.py --deal-id 34260    # Post specific deal
  python3 post_deal_safe.py --deal-id 34260 --skip-dupe-check  # Force post even if already posted
"""

import urllib.request
import urllib.error
import json
import subprocess
import sys
import os
import hashlib
import argparse
from datetime import datetime

# Config
DEALS_API = "https://www.gottahavecoups.com/api/v1/deals/latest?limit=200"
DEAL_API = "https://www.gottahavecoups.com/api/v1/deals/{deal_id}"

# Import deal_scorer for ranking
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
try:
    from deal_scorer import score_deal as scorer_score_deal, clean_title
    USE_DEAL_SCORER = True
except ImportError:
    USE_DEAL_SCORER = False

# Deal filtering criteria - TWO-TIER SYSTEM
PRIMARY_MIN_SCORE = 35            # Primary threshold (high quality)
SECONDARY_MIN_SCORE = 25          # Fallback threshold (medium quality)
MAX_PRICE = 500                   # Skip deals over $500
MIN_PRICE = 2                     # Skip deals under $2 (junk)
SKIP_CATEGORIES = [8]             # 8 = Books - nobody cares
SKIP_STORES = ['Costco']          # Requires membership
SHARE_IMAGE_URL = "https://www.gottahavecoups.com/api/v1/share/image/"
MAYA_HOST = "maya@maya"
POSTED_DB = "/home/crogers2287/openclaw-workspaces/ghc-marketing/posted_deals_v2.json"

def build_utm_url(deal_id, platform, campaign="daily_deals"):
    """Build URL with UTM tracking parameters"""
    base_url = f"https://gottahavecoups.com/deal/{deal_id}"
    utm_params = f"?utm_source={platform}&utm_medium=social&utm_campaign={campaign}&utm_content=deal_{deal_id}"
    return base_url + utm_params

def fetch_deals():
    """Fetch deals from API - pure Python, no shell"""
    try:
        req = urllib.request.Request(DEALS_API, headers={'User-Agent': 'Mozilla/5.0'})
        with urllib.request.urlopen(req, timeout=15) as resp:
            data = json.loads(resp.read())
            return data if isinstance(data, list) else data.get('deals', [])
    except Exception as e:
        print(f"Error fetching deals: {e}")
        return []

def fetch_deal_by_id(deal_id):
    """Fetch a specific deal by ID"""
    try:
        url = DEAL_API.format(deal_id=deal_id)
        req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
        with urllib.request.urlopen(req, timeout=15) as resp:
            return json.loads(resp.read())
    except Exception as e:
        print(f"Error fetching deal {deal_id}: {e}")
        return None

def score_deal(deal):
    """Score a deal using deal_scorer.py algorithm."""
    price = deal.get('price', 0) or 0
    category = deal.get('category_id')
    store = deal.get('store_name', '')

    # Hard disqualify
    if category in SKIP_CATEGORIES:
        return -1, []  # Books
    if store in SKIP_STORES:
        return -1, []  # Costco
    if price > MAX_PRICE or price < MIN_PRICE:
        return -1, []

    # Use deal_scorer if available
    if USE_DEAL_SCORER:
        score, reasons = scorer_score_deal(deal)
        return score, reasons

    # Fallback: simple scoring
    score = 0
    discount = deal.get('discount_percentage', 0) or 0
    if discount >= 50: score += 30
    elif discount >= 30: score += 20
    if 10 <= price <= 50: score += 20
    if store.lower() == 'amazon': score += 10
    return score, []

def filter_and_rank_deals(deals, min_score=None):
    """Filter out bad deals and rank by score using deal_scorer

    Args:
        deals: List of deals to filter
        min_score: Minimum score threshold (defaults to PRIMARY_MIN_SCORE)
    """
    if min_score is None:
        min_score = PRIMARY_MIN_SCORE

    scored = []
    for d in deals:
        s, reasons = score_deal(d)
        if s >= min_score:  # Only deals above minimum score
            scored.append((s, reasons, d))

    # Sort by score descending
    scored.sort(key=lambda x: x[0], reverse=True)
    return scored  # Returns (score, reasons, deal) tuples

def check_posted(deal_id, platform='x'):
    """Check if deal already posted - calls posted_db.py"""
    try:
        result = subprocess.run(
            ['python3', '/home/crogers2287/openclaw-workspaces/ghc-marketing/scripts/posted_db.py',
             'check', str(deal_id), platform],
            capture_output=True, text=True, timeout=10
        )
        return 'POSTED:YES' in result.stdout or 'POSTED:YES' in result.stderr
    except Exception as e:
        print(f"Error checking posted_db: {e}")
        return False

def smart_truncate(text, max_len=80):
    """Truncate text at word boundary, add ... if needed"""
    import re

    # Clean up common junk from titles
    text = text.split('~')[0].strip()  # Remove ~price suffix

    # Remove price prefix patterns like "6.75* | " or "5 | " at the START
    text = re.sub(r'^\d+\.?\d*\*?\s*\|\s*', '', text)

    # Remove trailing price patterns like "$28 + Free S/H" or "$55.20 + Free Shipping"
    text = re.sub(r'\s*\$[\d.]+\s*\+?\s*(Free\s*(S/H|Shipping))?$', '', text, flags=re.IGNORECASE)

    # Remove " at Amazon" or " at Walmart" suffix
    text = re.sub(r'\s+at\s+(Amazon|Walmart|Target|eBay)$', '', text, flags=re.IGNORECASE)

    text = text.strip()

    if len(text) <= max_len:
        return text

    # Find last space before max_len
    truncated = text[:max_len]
    last_space = truncated.rfind(' ')

    if last_space > max_len * 0.5:  # Only use space if it's not too early
        truncated = truncated[:last_space]

    return truncated.rstrip('.,;:-') + '...'

def generate_product_hashtags(title, category=None):
    """Generate product-specific hashtags based on deal content"""
    title_lower = title.lower()
    hashtags = []

    # Home & Kitchen
    if any(word in title_lower for word in ['wallpaper', 'wall art', 'decor', 'mirror', 'lamp', 'rug', 'curtain', 'furniture', 'shelf', 'storage ottoman', 'headboard']):
        hashtags.append('#HomeDecor')
    if any(word in title_lower for word in ['kitchen', 'cookware', 'utensil', 'appliance', 'blender', 'air fryer', 'pot', 'pan', 'knife', 'cutting board', 'spatula', 'bowl']):
        hashtags.append('#Kitchen')
    if any(word in title_lower for word in ['storage', 'organizer', 'container', 'bin', 'basket', 'drawer', 'closet', 'shelf']):
        hashtags.append('#Storage')
    if any(word in title_lower for word in ['bathroom', 'towel', 'shower', 'faucet', 'soap', 'toilet', 'vanity']):
        hashtags.append('#Bathroom')
    if any(word in title_lower for word in ['bedding', 'pillow', 'blanket', 'comforter', 'sheet', 'mattress']):
        hashtags.append('#Bedding')

    # Electronics
    if any(word in title_lower for word in ['headphone', 'earbud', 'speaker', 'charger', 'cable', 'laptop', 'tablet', 'phone', 'smartwatch', 'keyboard', 'mouse', 'monitor', 'tv', 'camera']):
        hashtags.append('#Tech')
    if any(word in title_lower for word in ['gaming', 'game', 'controller', 'console', 'playstation', 'xbox', 'nintendo']):
        hashtags.append('#Gaming')

    # Clothing & Fashion
    if any(word in title_lower for word in ['shirt', 'polo', 't-shirt', 'sweater', 'jacket', 'hoodie', 'blazer', 'dress', 'skirt']):
        hashtags.append('#Clothing')
    if any(word in title_lower for word in ['shoe', 'sneaker', 'boot', 'sandal', 'slipper']):
        hashtags.append('#Shoes')
    if any(word in title_lower for word in ['jewelry', 'necklace', 'bracelet', 'earring', 'ring', 'watch']):
        hashtags.append('#Jewelry')
    if any(word in title_lower for word in ['bag', 'backpack', 'purse', 'tote', 'luggage']):
        hashtags.append('#Bags')

    # Toys & Kids
    if any(word in title_lower for word in ['toy', 'plush', 'doll', 'action figure', 'lego', 'nerf', 'game', 'puzzle', 'jurassic', 'dinosaur']):
        hashtags.append('#Toys')
    if any(word in title_lower for word in ['baby', 'stroller', 'car seat', 'diaper', 'bottle', 'pacifier', 'nursery']):
        hashtags.append('#Baby')
    if any(word in title_lower for word in ['kids', 'boys', 'girls', 'children', 'toddler']):
        hashtags.append('#Kids')

    # Health & Beauty
    if any(word in title_lower for word in ['makeup', 'lipstick', 'mascara', 'foundation', 'skincare', 'serum', 'cream', 'lotion']):
        hashtags.append('#Beauty')
    if any(word in title_lower for word in ['vitamin', 'supplement', 'protein', 'fitness', 'gym', 'workout', 'yoga', 'exercise']):
        hashtags.append('#Fitness')

    # Outdoor & Garden
    if any(word in title_lower for word in ['outdoor', 'patio', 'garden', 'grill', 'lawn', 'planter', 'furniture set', 'hammock', 'umbrella']):
        hashtags.append('#Outdoor')
    if any(word in title_lower for word in ['tool', 'drill', 'saw', 'hammer', 'screwdriver', 'wrench', 'ladder']):
        hashtags.append('#Tools')

    # Automotive
    if any(word in title_lower for word in ['car', 'auto', 'tire', 'oil', 'battery', 'dash cam', 'seat cover', 'floor mat', 'degreaser', 'cleaner', 'wax']):
        hashtags.append('#Automotive')

    # Pet
    if any(word in title_lower for word in ['dog', 'cat', 'pet', 'treat', 'food bowl', 'leash', 'collar', 'bed', 'toy']):
        hashtags.append('#Pet')

    # Office & School
    if any(word in title_lower for word in ['desk', 'chair', 'office', 'pen', 'notebook', 'paper', 'stapler', 'printer', 'ink']):
        hashtags.append('#Office')
    if any(word in title_lower for word in ['school', 'backpack', 'lunchbox', 'supply', 'crayon', 'marker']):
        hashtags.append('#School')

    # Food & Drink
    if any(word in title_lower for word in ['coffee', 'tea', 'snack', 'candy', 'chocolate', 'protein bar']):
        hashtags.append('#Foodie')
    if any(word in title_lower for word in ['water bottle', 'tumbler', 'mug', 'cup', 'thermos']):
        hashtags.append('#Drinkware')

    # Books & Media
    if any(word in title_lower for word in ['book', 'novel', 'audiobook', 'kindle']):
        hashtags.append('#Books')

    return hashtags

def build_tweet(deal):
    """Build tweet text with proper $ signs - pure Python"""
    title = smart_truncate(deal.get('title', ''), max_len=80)
    price = deal.get('price', 0)
    savings = deal.get('savings') or 0
    orig_price = deal.get('original_price')  # Don't calculate fake original price
    discount = deal.get('discount_percentage', 0)
    deal_id = deal.get('id')

    # Calculate original price ONLY if we have real savings data
    if not orig_price and savings and savings > 0:
        orig_price = price + savings

    # Format price with $ sign (SAFE - no shell involved)
    price_str = f"${price:.2f}" if price else ""
    orig_price_str = f"${orig_price:.2f}" if orig_price else ""

    # Validate: original price should be HIGHER than current price
    if orig_price and orig_price <= price:
        orig_price = None  # Don't show bogus "was" price
        orig_price_str = ""

    # Build tweet - vary the hook so posts don't all look the same
    import random
    if discount and discount >= 50:
        hooks_big_discount = [
            f"🔥 {int(discount)}% OFF!",
            f"💥 {int(discount)}% off — seriously!",
            f"⚡ {int(discount)}% off right now!",
            f"🚨 {int(discount)}% OFF — don't sleep on this!",
            f"Whoa — {int(discount)}% off!",
            f"👀 {int(discount)}% off. You seeing this?",
        ]
        hook = random.choice(hooks_big_discount)
    elif price and price < 10:
        hooks_under10 = [
            "🔥 UNDER $10!",
            "💰 Under $10 — go go go!",
            "🤑 Less than $10!",
            "👀 Under ten bucks!",
            "⚡ Sub-$10 steal!",
            "This one's under $10 👀",
        ]
        hook = random.choice(hooks_under10)
    elif price and price < 20:
        hooks_cheap = [
            "💰 Solid deal!",
            "🙌 Steal alert!",
            "✨ Nice find!",
            "👀 Look at this price!",
            "🎯 Deal of the day!",
        ]
        hook = random.choice(hooks_cheap)
    else:
        hooks_general = [
            "🔥 Hot find!",
            "💥 Deal alert!",
            "⚡ Check this out!",
            "🎯 Spotted a good one!",
            "👀 Worth a look!",
            "🙌 Don't miss this!",
            "✨ Nice price drop!",
            "🚨 Price drop alert!",
            "💰 Save on this one!",
            "🛒 Just spotted this!",
            "🤩 Look what I found!",
        ]
        hook = random.choice(hooks_general)

    tweet_parts = [hook, title]

    # Only show "was X" if we have a REAL higher original price
    if orig_price and orig_price > price:
        tweet_parts.append(f"just {price_str} (was {orig_price_str}) — {int(discount)}% OFF!")
    elif discount:
        tweet_parts.append(f"just {price_str} — {int(discount)}% OFF!")
    elif price:
        tweet_parts.append(f"just {price_str}!")

    # Add hashtags - product-specific + standard deal hashtags
    product_tags = generate_product_hashtags(title, deal.get('category_id'))
    base_tags = ["#DailyDeals", "#Savings", "#DealAlert", "#GottaHaveCoups"]

    # Combine product tags first (more relevant), then base tags
    all_tags = product_tags + base_tags
    hashtags = " ".join(all_tags[:5])  # Limit to 5 hashtags to keep tweet clean

    tweet_parts.append(hashtags)

    # Link LAST
    # Use tracked URL for X/Twitter
    tweet_parts.append(build_utm_url(deal_id, "twitter"))

    tweet = "\n\n".join(tweet_parts)

    # Validate length
    if len(tweet) > 280:
        tweet = tweet[:277] + "..."

    return tweet

def validate_tweet(tweet):
    """Validate tweet has no shell artifacts or broken prices"""
    errors = []

    if '/bin/bash' in tweet:
        errors.append("Shell artifact: /bin/bash")
    if 'python3 -c' in tweet:
        errors.append("Shell artifact: python3 -c")

    # Check for prices without $ (look for decimal patterns)
    import re
    # Pattern: number followed by decimal but no $ before it
    # This is a simple check - prices should have $ in front

    if len(tweet) > 280:
        errors.append(f"Too long: {len(tweet)} chars")

    if '�' in tweet:
        errors.append("Broken emoji detected")

    return errors

def download_image(deal_id, deal=None):
    """Download product image from deal's image_url (NOT share endpoint)"""
    img_path = f"/tmp/deal_{deal_id}.jpg"
    try:
        # CRITICAL: Use the deal's actual image_url, NOT the share endpoint
        # The share endpoint was serving cached/wrong images
        if deal and deal.get('image_url'):
            image_url = deal['image_url']
            print(f"  Using deal image: {image_url}")
        else:
            # Fallback to share endpoint only if no deal provided (shouldn't happen)
            image_url = f"{SHARE_IMAGE_URL}{deal_id}"
            print(f"  WARNING: Using share endpoint (deal object missing)")

        req = urllib.request.Request(image_url, headers={'User-Agent': 'Mozilla/5.0'})
        with urllib.request.urlopen(req, timeout=15) as resp, open(img_path, 'wb') as f:
            f.write(resp.read())
        return img_path
    except Exception as e:
        print(f"Error downloading image: {e}")
        return None

def post_to_x(deal_id, tweet_text, img_path):
    """Post to X via Maya VM twikit - direct method"""
    # Copy image to Maya
    try:
        subprocess.run(['scp', '-q', img_path, f"{MAYA_HOST}:/home/maya/workspace/downloads/deal_{deal_id}.jpg"],
                      check=True, timeout=30)
    except subprocess.CalledProcessError as e:
        print(f"SCP failed: {e}")
        return None

    # Create Python script for Maya to execute
    maya_script = f'''
import asyncio
import json
from twikit import Client

async def post():
    client = Client()
    with open("/home/maya/workspace/x_cookies.json") as f:
        cookies = json.load(f)
    client.set_cookies(cookies)

    tweet_text = {repr(tweet_text)}

    media_id = await client.upload_media("/home/maya/workspace/downloads/deal_{deal_id}.jpg")
    print(f"Media: {{media_id}}")

    result = await client.create_tweet(tweet_text, media_ids=[media_id])
    print(f"SUCCESS Tweet: {{result.id}}")

asyncio.run(post())
'''

    # Write script to temp file and copy to Maya
    script_path = f"/tmp/post_{deal_id}.py"
    with open(script_path, 'w') as f:
        f.write(maya_script)

    try:
        subprocess.run(['scp', '-q', script_path, f"{MAYA_HOST}:/home/maya/workspace/post_deal.py"],
                      check=True, timeout=10)
    except subprocess.CalledProcessError as e:
        print(f"Script copy failed: {e}")
        return None

    # Execute on Maya
    try:
        result = subprocess.run(
            ['ssh', MAYA_HOST, 'cd /home/maya/workspace && python3 post_deal.py'],
            capture_output=True, text=True, timeout=60
        )
        print(result.stdout)

        # Extract tweet ID
        for line in result.stdout.split('\n'):
            if 'SUCCESS Tweet:' in line:
                return line.split('SUCCESS Tweet:')[1].strip()

        if result.returncode != 0 and result.stderr:
            # Filter out SSH warnings
            errors = [l for l in result.stderr.split('\n') if 'Warning' not in l and l.strip()]
            if errors:
                print(f"Error: {errors[0][:200]}")
        return None
    except Exception as e:
        print(f"Error posting to X: {e}")
        return None

def post_to_pinterest(deal_id, deal):
    """Post to Pinterest via Maya VM py3-pinterest"""
    title = deal.get('title', '').split('~')[0].strip()[:100]
    description = f"Deal from GottaHaveCoups - {title}"
    board_id = "1134485031073239531"  # Today's Best Deals
    # Use UTM tracked URL for Pinterest
    pin_url = build_utm_url(deal_id, "pinterest")

    cmd = [
        'ssh', MAYA_HOST,
        f'cd /home/maya/workspace && python3 scripts/pin_post_v2.py {deal_id} "{title}" "{description}" {board_id} "{pin_url}"'
    ]

    try:
        result = subprocess.run(cmd, capture_output=True, text=True, timeout=60)
        print(result.stdout)
        return result.returncode == 0
    except Exception as e:
        print(f"Error posting to Pinterest: {e}")
        return False

def notify_group_for_facebook(deal_id, deal, tweet_text, tweet_id, img_path):
    """Send Chelsea branded deal image + tweet text + FB UTM link via Telegram."""
    try:
        import urllib.parse
        deal_url = "https://gottahavecoups.com/deal/" + str(deal_id)
        fb_url = deal_url + "?utm_source=facebook&utm_medium=social&utm_campaign=deals"
        # Strip Twitter UTM link from tweet text before sending to Chelsea
        import re as _re
        clean_tweet = _re.sub(r'https://gottahavecoups\.com/deal/\S*utm_source=twitter\S*', '', tweet_text).strip()
        caption = clean_tweet + "\n\n" + fb_url
        BOT_TOKEN = "8536275460:AAH08AuOKhs_2uEWzX1HlyKMdMRZ-AZLqa8"
        CHAT_ID = "-5121721159"
        TG_API = "https://api.telegram.org/bot" + BOT_TOKEN
        share_img_path = "/tmp/deal_" + str(deal_id) + "_share.jpg"
        try:
            req = urllib.request.Request(SHARE_IMAGE_URL + str(deal_id), headers={"User-Agent": "Mozilla/5.0"})
            with urllib.request.urlopen(req, timeout=15) as resp, open(share_img_path, "wb") as fout:
                fout.write(resp.read())
            img_to_send = share_img_path
        except Exception:
            img_to_send = img_path
        if img_to_send and os.path.exists(img_to_send):
            with open(img_to_send, "rb") as imgf:
                img_data = imgf.read()
            bd = "TGBound"
            sep = "--" + bd + "\r\n"
            body = (
                (sep + 'Content-Disposition: form-data; name="chat_id"\r\n\r\n' + CHAT_ID + "\r\n").encode()
                + (sep + 'Content-Disposition: form-data; name="caption"\r\n\r\n' + caption + "\r\n").encode()
                + (sep + 'Content-Disposition: form-data; name="photo"; filename="deal.jpg"\r\nContent-Type: image/jpeg\r\n\r\n').encode()
                + img_data
                + ("\r\n--" + bd + "--\r\n").encode()
            )
            req = urllib.request.Request(
                TG_API + "/sendPhoto", data=body,
                headers={"Content-Type": "multipart/form-data; boundary=" + bd}
            )
            with urllib.request.urlopen(req, timeout=20) as resp:
                result = json.loads(resp.read())
                print("Group notify: " + ("OK" if result.get("ok") else str(result)))
        else:
            data = urllib.parse.urlencode({"chat_id": CHAT_ID, "text": caption}).encode()
            urllib.request.urlopen(urllib.request.Request(TG_API + "/sendMessage", data=data), timeout=15)
            print("Text-only notify sent")
    except Exception as e:
        print("Group notify error: " + str(e))

def record_post(deal_id, platform, post_id, url):
    """Record post in database"""
    try:
        subprocess.run([
            'python3', '/home/crogers2287/openclaw-workspaces/ghc-marketing/scripts/posted_db.py',
            'record', str(deal_id), platform, str(post_id), url
        ], check=True, timeout=10)
        return True
    except Exception as e:
        print(f"Error recording post: {e}")
        return False

def check_priority_queue():
    """Check if there's a priority queue and pop the next deal ID"""
    queue_file = "/home/crogers2287/openclaw-workspaces/ghc-marketing/config/priority_queue.json"

    if not os.path.exists(queue_file):
        return None

    try:
        with open(queue_file, 'r') as f:
            data = json.load(f)

        queue = data.get('queue', [])
        if not queue:
            return None

        # Pop first deal from queue
        next_deal_id = queue.pop(0)

        # Save updated queue
        data['queue'] = queue
        with open(queue_file, 'w') as f:
            json.dump(data, f, indent=2)

        print(f"📋 Priority queue: Using deal {next_deal_id} ({len(queue)} remaining)")
        return next_deal_id
    except Exception as e:
        print(f"Error reading priority queue: {e}")
        return None

def main():
    # Parse arguments
    parser = argparse.ArgumentParser(description='GHC Deal Poster')
    parser.add_argument('--deal-id', type=int, help='Specific deal ID to post')
    parser.add_argument('--skip-dupe-check', action='store_true', help='Skip duplicate check (force post)')
    parser.add_argument('--x-only', action='store_true', help='Post to X only, skip Pinterest')
    parser.add_argument('--pinterest-only', action='store_true', help='Post to Pinterest only, skip X')
    args = parser.parse_args()

    print("=" * 60)
    print("GHC Deal Poster - Starting")
    print("=" * 60)

    # Check priority queue first (unless --deal-id was explicitly provided)
    if not args.deal_id:
        priority_deal_id = check_priority_queue()
        if priority_deal_id:
            args.deal_id = priority_deal_id
            args.skip_dupe_check = True  # Skip dupe check for priority queue items

    if args.deal_id:
        # Fetch specific deal
        print(f"\n[1/6] Fetching deal {args.deal_id}...")
        selected_deal = fetch_deal_by_id(args.deal_id)
        if not selected_deal:
            print(f"Deal {args.deal_id} not found. Exiting.")
            return 1
        print(f"Found: {selected_deal.get('title', '')[:60]}")

        # Check if already posted (unless skip flag)
        if not args.skip_dupe_check and check_posted(args.deal_id, 'x'):
            print(f"Deal {args.deal_id} already posted to X. Use --skip-dupe-check to force.")
            return 0
    else:
        # Auto-select from feed with quality filtering
        print("\n[1/6] Fetching deals...")
        deals = fetch_deals()
        if not deals:
            print("No deals fetched. Exiting.")
            return 1
        print(f"Fetched {len(deals)} deals")

        # TWO-TIER FILTERING SYSTEM
        # Try primary threshold first (high quality)
        print("\n[2/6] Filtering and ranking deals (PRIMARY_MIN_SCORE={})...".format(PRIMARY_MIN_SCORE))
        ranked_deals = filter_and_rank_deals(deals, PRIMARY_MIN_SCORE)
        print(f"After primary filtering: {len(ranked_deals)} high-quality deals")

        # Find first unposted deal from ranked list
        selected_deal = None
        selected_score = 0
        selected_reasons = []
        tier_used = "PRIMARY"

        for score, reasons, deal in ranked_deals:
            d_id = deal.get('id')
            if not check_posted(d_id, 'x'):
                selected_deal = deal
                selected_score = score
                selected_reasons = reasons
                print(f"✓ Selected deal {d_id} from PRIMARY tier (score: {score}/100)")
                print(f"  → {', '.join(reasons[:3])}")
                print(f"  {deal.get('title', '')[:60]}")
                break

        # FALLBACK: If no primary deals, try secondary threshold
        if not selected_deal:
            print(f"\n⚠  No unposted deals in primary tier. Trying SECONDARY tier (MIN_SCORE={SECONDARY_MIN_SCORE})...")
            ranked_deals = filter_and_rank_deals(deals, SECONDARY_MIN_SCORE)
            print(f"After secondary filtering: {len(ranked_deals)} medium-quality deals")

            for score, reasons, deal in ranked_deals:
                d_id = deal.get('id')
                if not check_posted(d_id, 'x'):
                    selected_deal = deal
                    selected_score = score
                    selected_reasons = reasons
                    tier_used = "SECONDARY"
                    print(f"✓ Selected deal {d_id} from SECONDARY tier (score: {score}/100)")
                    print(f"  → {', '.join(reasons[:3])}")
                    print(f"  {deal.get('title', '')[:60]}")
                    break

        if not selected_deal:
            print("❌ No unposted deals in either tier. Exiting.")
            return 0

    deal_id = selected_deal.get('id')

    # 3. Build tweet
    print("\n[3/6] Building tweet...")
    tweet = build_tweet(selected_deal)
    print(f"Tweet:\n{tweet}\n")

    # 4. Validate
    print("[4/6] Validating...")
    errors = validate_tweet(tweet)
    if errors:
        print(f"VALIDATION FAILED: {errors}")
        return 1
    print("✓ Validation passed")

    # 5. Download image
    print("\n[5/6] Downloading image...")
    img_path = download_image(deal_id, selected_deal)
    if not img_path:
        print("Failed to download image. Continuing without image...")
    else:
        print(f"✓ Image saved: {img_path}")

    # 6. Post to X
    tweet_id = None
    if not args.pinterest_only:
        print("\n[6/6] Posting to X...")
        tweet_id = post_to_x(deal_id, tweet, img_path or f"/tmp/deal_{deal_id}.jpg")
        if not tweet_id:
            print("Failed to post to X")
            if not args.x_only:  # Continue to Pinterest if X fails
                pass
            else:
                return 1
        else:
            print(f"✓ Posted to X: {tweet_id}")
            record_post(deal_id, 'x', tweet_id, f"https://x.com/GottaHaveCoups/status/{tweet_id}")
            # Notify group chat with FB-ready package for Chelsea
            notify_group_for_facebook(deal_id, selected_deal, tweet, tweet_id, img_path)

    # 7. Post to Pinterest
    if not args.x_only:
        print("\n[Bonus] Posting to Pinterest...")
        if post_to_pinterest(deal_id, selected_deal):
            print("✓ Posted to Pinterest")
            record_post(deal_id, 'pinterest', 'pinned', f"https://pinterest.com/GottaHaveCoups")
        else:
            print("⚠ Pinterest post failed (non-critical)")

    print("\n" + "=" * 60)
    print("Done!")
    print("=" * 60)
    return 0

if __name__ == '__main__':
    sys.exit(main())


The posting script looks correct - it's using the deal's `image_url` directly from the deal object, not the share endpoint (except as a fallback when the deal object is missing).

The problem must be upstream in the **scraper** that provides the image URLs. Let me check the scraper tasks and see how image URLs are being assigned to deals.

Let me look at the scraper_tasks.py to understand how deals get their image URLs.

ssh carl "cat /opt/ghc20/backend/app/tasks/scraper_tasks.py | head -200"


Warning: Permanently added 'carl' (ED25519) to the list of known hosts.
"""Scraper background tasks"""
import asyncio
import logging
import sys
from datetime import datetime, timedelta
from typing import Dict, Any, Optional
from pathlib import Path

from sqlalchemy import select, delete
from app.core.celery_app import celery_app
from app.db.session import get_celery_session_factory
from app.models.deal import Deal as DealModel
from app.models.category import Category as CategoryModel
from app.services.content_cleaner import content_cleaner

# Add project root to path for scrapers module
project_root = Path(__file__).parent.parent.parent.parent
if str(project_root) not in sys.path:
    sys.path.insert(0, str(project_root))

logger = logging.getLogger(__name__)

def _serialize_deal_data(deal_data: Dict[str, Any]) -> Dict[str, Any]:
    """Convert datetime objects to ISO strings for JSON serialization."""
    serialized = {}
    for key, value in deal_data.items():
        if isinstance(value, datetime):
            serialized[key] = value.isoformat()
        elif isinstance(value, dict):
            serialized[key] = _serialize_deal_data(value)
        elif isinstance(value, list):
            serialized[key] = [
                _serialize_deal_data(item) if isinstance(item, dict)
                else item.isoformat() if isinstance(item, datetime)
                else item
                for item in value
            ]
        else:
            serialized[key] = value
    return serialized

@celery_app.task(name="app.tasks.scraper_tasks.scrape_and_process_deals")
def scrape_and_process_deals(quick_run: bool = False) -> Dict[str, Any]:
    """
    Scrape deals from RSS feeds and process through pipeline.

    Args:
        quick_run: If True, only scrape high-priority feeds

    Returns:
        Statistics dictionary
    """
    return asyncio.run(_scrape_and_process_deals_async(quick_run))

async def _scrape_and_process_deals_async(quick_run: bool = False) -> Dict[str, Any]:
    """Async implementation of scrape_and_process_deals"""
    logger.info(f"Starting deal scraping task (quick_run={quick_run})")
    start_time = datetime.now()

    # Create a fresh session factory for this Celery task
    CelerySessionLocal = get_celery_session_factory()

    # Check if scraping is enabled and get pipeline setting
    pipeline_type = 'old'  # default to RSS
    async with CelerySessionLocal() as session:
        from app.models.scraper_settings import ScraperSettings as ScraperSettingsModel

        result = await session.execute(
            select(ScraperSettingsModel).where(ScraperSettingsModel.id == 1)
        )
        settings = result.scalar_one_or_none()

        if settings and not settings.enabled:
            logger.info("Scraper is disabled in settings - skipping run")
            return {
                'skipped': True,
                'reason': 'Scraper disabled in settings',
                'timestamp': datetime.now().isoformat(),
            }

        # Get pipeline setting
        if settings:
            pipeline_type = getattr(settings, 'pipeline', 'old') or 'old'

        # Get camel min discount threshold from aggregator config (not Django settings)
        camel_min_discount = 30.0  # Default to 30%
        try:
            import json
            from pathlib import Path
            config_path = Path(__file__).parent.parent.parent / "aggregator_config.json"
            if config_path.exists():
                with open(config_path) as f:
                    agg_config = json.load(f)
                    camel_min_discount = agg_config.get('camel_config', {}).get('min_savings', 30.0)
                    logger.info(f"Loaded camel_min_discount from aggregator_config.json: {camel_min_discount}%")
        except Exception as e:
            logger.warning(f"Could not load aggregator config, using default: {e}")

    logger.info(f"Using pipeline: {pipeline_type}")

    try:
        # Import here to avoid circular imports
        import sys
        import os
        from pathlib import Path

        # Add both backend and project root to path
        backend_dir = Path(__file__).parent.parent.parent  # /home/user/ghc20/backend
        project_root = backend_dir.parent  # /home/user/ghc20

        # Add project root first so 'scrapers' module can be found
        if str(project_root) not in sys.path:
            sys.path.insert(0, str(project_root))

        # Add scrapers directory as well
        scrapers_path = project_root / "scrapers"
        if str(scrapers_path) not in sys.path:
            sys.path.insert(0, str(scrapers_path))

        # Get category mapping
        category_map = await _get_category_mapping()

        if pipeline_type == 'aggregator':
            # Use the new aggregator pipeline (Camel/Slick/Hip2Save)
            # NOW USES SAME PROCESSING AS OLD PIPELINE: URL extraction, affiliate conversion, local image download
            logger.info(f"Using aggregator pipeline (Camel/Slick/Hip2Save) with FULL processing (camel_min_discount={camel_min_discount}%)")
            from app.services.rss_aggregator import get_aggregator_manager
            from app.services.rss_aggregator.base import BaseScraper
            from scrapers.categorizer.categorizer import DealCategorizer
            from scrapers.quality_scorer.scorer import DealQualityScorer

            # Import processing utilities from RSS pipeline
            from scrapers.utils import AffiliateConverter, MultiRetailerExtractor
            from app.services.image_downloader import ImageDownloader

            manager = await get_aggregator_manager()
            aggregated_deals = await manager.get_all_deals(
                limit=100,
                min_savings=camel_min_discount,  # Use the setting for CamelCamelCamel threshold
                use_cache=False
            )
            logger.info(f"Fetched {len(aggregated_deals)} raw deals from aggregators")

            # Load feed category mappings from aggregator config
            feed_category_mappings = manager.config.get('feed_category_mappings', {})
            logger.info(f"Feed category mappings: {feed_category_mappings}")

            # Step 1: Extract images from deal pages
            logger.info("Step 1: Extracting images from deal pages...")
            from app.services.rss_aggregator.camel import CamelScraper
            image_scraper = CamelScraper()
            aggregated_deals = await image_scraper.enrich_deals_with_images(aggregated_deals, max_concurrent=3)
            await image_scraper.close()
            images_found = sum(1 for d in aggregated_deals if d.get('image'))
            logger.info(f"Extracted images for {images_found}/{len(aggregated_deals)} deals")

            # Step 2: Initialize processors (SAME as old pipeline)
            categorizer = DealCategorizer()
            quality_scorer = DealQualityScorer()
            affiliate_converter = AffiliateConverter()
            multi_retailer_extractor = MultiRetailerExtractor()
            image_downloader = ImageDownloader()

            # Step 3: Process deals through the SAME pipeline as RSS feeds
            logger.info("Step 3: Processing deals through full pipeline (URL extraction, affiliates, images)...")
            processed_deals = []
            urls_extracted = 0
            local_images = 0

            for deal_data in aggregated_deals:
                title = deal_data.get('title', '')
                description = deal_data.get('description', '')
                source = deal_data.get('source', '')  # e.g., "Slickdeals - Black Friday 2025"
                link = deal_data.get('link', '')  # SlickDeals/Hip2Save page URL

                # Determine feed key for category mapping (e.g., "slick_41" from "Slickdeals - Black Friday 2025")
                feed_key = None
                if 'slickdeals' in source.lower():
                    forum_id = deal_data.get('forum_id')
                    if forum_id:
                        feed_key = f"slick_{forum_id}"
                elif 'hip2save' in source.lower():
                    feed_key = deal_data.get('config_key')  # Hip2Save has config_key

                # Check for feed category override FIRST (takes priority over AI)
                category_override = feed_category_mappings.get(feed_key) if feed_key else None

                # Extract actual retailer URL and affiliate link
                # For CamelCamelCamel, use the generated affiliate_link
                # For SlickDeals/Hip2Save, we need to extract the actual retailer URL
                retailer_url = None
                affiliate_link = None
                store_name = None

                if deal_data.get('asin'):
                    # CamelCamelCamel deal - already has affiliate link
                    affiliate_link = deal_data.get('affiliate_link', '')


The scraper uses an image downloader that stores images based on content hash. The issue seems to be upstream - either the scraper is extracting the wrong image URL from the source pages, OR there's a bug in the image enrichment step.

Let me also check the share image service to see if there's a caching issue there that could cause the wrong image to be served. I already cleared the cache but I want to understand if there's a deeper bug.

Actually, looking at the code flow:
1. Scraper extracts image from source page
2. ImageDownloader saves it with content-based hash
3. Deal gets `image_url` pointing to `/api/v1/deal-images/{hash}.jpg`
4. Posting script downloads from `deal['image_url']`
5. X posts the image

The bug could be:
1. Scraper extracting wrong image from source page (upstream bug)
2. Image cache serving wrong image (cleared this)
3. X's card preview caching old images (external)

Let me check if the share image service has any bugs that could cause it to return the wrong image for a deal ID. Also, let me add some validation to the posting script to detect when an image doesn't match the deal.

Let me check the share image service to see if there's a bug in how it fetches deal images:

ssh carl "cat /opt/ghc20/backend/app/services/image_watermark.py | head -150"


Warning: Permanently added 'carl' (ED25519) to the list of known hosts.
"""Image watermarking service for social sharing"""
import io
import logging
import hashlib
from pathlib import Path
from typing import Optional, Tuple

import httpx
from PIL import Image, ImageDraw, ImageFont

logger = logging.getLogger(__name__)

# Paths
PROJECT_ROOT = Path(__file__).parent.parent.parent.parent  # /home/user/ghc20
LOGO_PATH = PROJECT_ROOT / "frontend" / "public" / "images" / "logotrans.png"
CACHE_DIR = PROJECT_ROOT / "frontend" / "public" / "share-images"

# Watermark settings
WATERMARK_OPACITY = 180  # 0-255 (180 = ~70% opacity)
WATERMARK_POSITION = "bottom-right"  # bottom-right, bottom-left, top-right, top-left
WATERMARK_PADDING = 15  # pixels from edge
WATERMARK_SCALE = 0.15  # Logo size as percentage of image width (15%)

class ImageWatermarkService:
    """Service for adding watermarks to deal images for social sharing"""

    def __init__(self):
        self.logo: Optional[Image.Image] = None
        self._load_logo()

        # Ensure cache directory exists
        CACHE_DIR.mkdir(parents=True, exist_ok=True)

    def _load_logo(self) -> None:
        """Load the watermark logo"""
        try:
            if LOGO_PATH.exists():
                self.logo = Image.open(LOGO_PATH).convert("RGBA")
                logger.info(f"Loaded watermark logo: {LOGO_PATH}")
            else:
                logger.warning(f"Watermark logo not found: {LOGO_PATH}")
        except Exception as e:
            logger.error(f"Failed to load watermark logo: {e}")

    def _get_cache_path(self, image_url: str) -> Path:
        """Generate cache path for watermarked image"""
        url_hash = hashlib.md5(image_url.encode()).hexdigest()[:16]
        return CACHE_DIR / f"wm_{url_hash}.png"

    async def fetch_image(self, image_url: str) -> Optional[Image.Image]:
        """Fetch image from URL or local path"""
        try:
            # Handle local file paths (start with / or are relative paths like /deal-images/...)
            if image_url.startswith('/') and not image_url.startswith('//'):
                # Local path - resolve to frontend public directory
                local_path = PROJECT_ROOT / "frontend" / "public" / image_url.lstrip('/')
                if local_path.exists():
                    logger.debug(f"Loading local image: {local_path}")
                    return Image.open(local_path).convert("RGBA")
                else:
                    logger.warning(f"Local image not found: {local_path}")
                    return None

            # HTTP/HTTPS URL
            async with httpx.AsyncClient(timeout=10.0) as client:
                response = await client.get(image_url, follow_redirects=True)
                response.raise_for_status()

                image_data = io.BytesIO(response.content)
                return Image.open(image_data).convert("RGBA")
        except Exception as e:
            logger.error(f"Failed to fetch image {image_url}: {e}")
            return None

    def add_watermark(
        self,
        image: Image.Image,
        position: str = WATERMARK_POSITION,
        opacity: int = WATERMARK_OPACITY,
        scale: float = WATERMARK_SCALE,
        padding: int = WATERMARK_PADDING,
    ) -> Image.Image:
        """Add watermark to image"""
        if not self.logo:
            logger.warning("No logo loaded, returning original image")
            return image

        # Calculate watermark size based on image width
        img_width, img_height = image.size
        watermark_width = int(img_width * scale)

        # Maintain aspect ratio
        logo_aspect = self.logo.width / self.logo.height
        watermark_height = int(watermark_width / logo_aspect)

        # Resize logo
        watermark = self.logo.resize(
            (watermark_width, watermark_height),
            Image.Resampling.LANCZOS
        )

        # Adjust opacity
        if opacity < 255:
            watermark = self._adjust_opacity(watermark, opacity)

        # Calculate position
        x, y = self._calculate_position(
            image.size,
            watermark.size,
            position,
            padding
        )

        # Create output image
        output = image.copy()

        # Paste watermark with transparency
        output.paste(watermark, (x, y), watermark)

        return output

    def _adjust_opacity(self, image: Image.Image, opacity: int) -> Image.Image:
        """Adjust image opacity"""
        # Split into channels
        r, g, b, a = image.split()

        # Adjust alpha channel
        a = a.point(lambda x: int(x * opacity / 255))

        # Merge back
        return Image.merge("RGBA", (r, g, b, a))

    def _calculate_position(
        self,
        image_size: Tuple[int, int],
        watermark_size: Tuple[int, int],
        position: str,
        padding: int
    ) -> Tuple[int, int]:
        """Calculate watermark position"""
        img_w, img_h = image_size
        wm_w, wm_h = watermark_size

        positions = {
            "bottom-right": (img_w - wm_w - padding, img_h - wm_h - padding),
            "bottom-left": (padding, img_h - wm_h - padding),
            "top-right": (img_w - wm_w - padding, padding),
            "top-left": (padding, padding),
            "center": ((img_w - wm_w) // 2, (img_h - wm_h) // 2),


I see the issue! The share image service caches based on the **image URL hash** (line 48: `url_hash = hashlib.md5(image_url.encode()).hexdigest()[:16]`). This is correct - it caches by the original image URL.

But the problem might be that X (Twitter) has its own card preview cache that's separate from our cache. When X fetches the OG image URL, it caches that image for its card preview. If multiple deals share the same OG image URL (even briefly), they'd all show the same image.

But wait - I already confirmed there are no duplicate image URLs in the deals. So that's not it.

Let me think about this differently. Chelsea said it was "all the Macy's deals" and the hoodie. Let me check if there's something specific about those deals that could cause image mismatches.

Actually, I think I should add **validation** to the posting script to verify the downloaded image matches the deal. One simple check: download the image and verify its content hash against what we expect. Or at minimum, log the image dimensions/format so we can detect anomalies.

Let me also check the scraper to see if there's a bug in how it extracts images from deal pages.

Let me add **image validation** to the posting script to catch wrong images before they're posted. This will:
1. Download the image
2. Compare its content hash to detect duplicates/wrong images
3. Skip posting if the image looks suspicious


#!/usr/bin/env python3
"""
GHC Deal Poster - End-to-end deal posting script
NO SHELL COMMANDS WITH $ - Pure Python to avoid variable expansion bugs

Usage:
  python3 post_deal_safe.py                    # Auto-select from feed
  python3 post_deal_safe.py --deal-id 34260    # Post specific deal
  python3 post_deal_safe.py --deal-id 34260 --skip-dupe-check  # Force post even if already posted
"""

import urllib.request
import urllib.error
import json
import subprocess
import sys
import os
import hashlib
import argparse
from datetime import datetime

# Config
DEALS_API = "https://www.gottahavecoups.com/api/v1/deals/latest?limit=200"
DEAL_API = "https://www.gottahavecoups.com/api/v1/deals/{deal_id}"

# Import deal_scorer for ranking
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
try:
    from deal_scorer import score_deal as scorer_score_deal, clean_title
    USE_DEAL_SCORER = True
except ImportError:
    USE_DEAL_SCORER = False

# Deal filtering criteria - TWO-TIER SYSTEM
PRIMARY_MIN_SCORE = 35            # Primary threshold (high quality)
SECONDARY_MIN_SCORE = 25          # Fallback threshold (medium quality)
MAX_PRICE = 500                   # Skip deals over $500
MIN_PRICE = 2                     # Skip deals under $2 (junk)
SKIP_CATEGORIES = [8]             # 8 = Books - nobody cares
SKIP_STORES = ['Costco']          # Requires membership
SHARE_IMAGE_URL = "https://www.gottahavecoups.com/api/v1/share/image/"
MAYA_HOST = "maya@maya"
POSTED_DB = "/home/crogers2287/openclaw-workspaces/ghc-marketing/posted_deals_v2.json"

def build_utm_url(deal_id, platform, campaign="daily_deals"):
    """Build URL with UTM tracking parameters"""
    base_url = f"https://gottahavecoups.com/deal/{deal_id}"
    utm_params = f"?utm_source={platform}&utm_medium=social&utm_campaign={campaign}&utm_content=deal_{deal_id}"
    return base_url + utm_params

[661 more lines in file. Use offset=51 to continue.]


I need to add image validation to the posting script. The key checks I should add:
1. Track image content hashes to detect if multiple deals are using the same image (which would indicate a scraper bug)
2. Skip posting if the image has already been used for a different deal (duplicate image detection)
3. Add a "known bad image" blacklist for images that have been reported as wrong

Let me create a simple image tracking system:

edit

SHARE_IMAGE_URL = "https://www.gottahavecoups.com/api/v1/share/image/"
MAYA_HOST = "maya@maya"
POSTED_DB = "/home/crogers2287/openclaw-workspaces/ghc-marketing/posted_deals_v2.json"
IMAGE_HASH_DB = "/home/crogers2287/openclaw-workspaces/ghc-marketing/config/image_hashes.json"  # Track image hashes to detect duplicates


Successfully replaced text in /home/crogers2287/openclaw-workspaces/ghc-marketing/scripts/post_deal_safe.py.

     ...
  41 SKIP_STORES = ['Costco']          # Requires membership
  42 SHARE_IMAGE_URL = "https://www.gottahavecoups.com/api/v1/share/image/"
  43 MAYA_HOST = "maya@maya"
  44 POSTED_DB = "/home/crogers2287/openclaw-workspaces/ghc-marketing/posted_deals_v2.json"
+ 45 IMAGE_HASH_DB = "/home/crogers2287/openclaw-workspaces/ghc-marketing/config/image_hashes.json"  # Track image hashes to detect duplicates
  45
  46 def build_utm_url(deal_id, platform, campaign="daily_deals"):
  47     """Build URL with UTM tracking parameters"""
  48     base_url = f"https://gottahavecoups.com/deal/{deal_id}"
     ...


Now I need to add functions to:
1. Calculate the content hash of a downloaded image
2. Check if this image has been used before for a different deal
3. Record the image hash after successful postingLet me find the download_image function and add the validation.


    # Toys & Kids
    if any(word in title_lower for word in ['toy', 'plush', 'doll', 'action figure', 'lego', 'nerf', 'game', 'puzzle', 'jurassic', 'dinosaur']):
        hashtags.append('#Toys')
    if any(word in title_lower for word in ['baby', 'stroller', 'car seat', 'diaper', 'bottle', 'pacifier', 'nursery']):
        hashtags.append('#Baby')
    if any(word in title_lower for word in ['kids', 'boys', 'girls', 'children', 'toddler']):
        hashtags.append('#Kids')

    # Health & Beauty
    if any(word in title_lower for word in ['makeup', 'lipstick', 'mascara', 'foundation', 'skincare', 'serum', 'cream', 'lotion']):
        hashtags.append('#Beauty')
    if any(word in title_lower for word in ['vitamin', 'supplement', 'protein', 'fitness', 'gym', 'workout', 'yoga', 'exercise']):
        hashtags.append('#Fitness')

    # Outdoor & Garden
    if any(word in title_lower for word in ['outdoor', 'patio', 'garden', 'grill', 'lawn', 'planter', 'furniture set', 'hammock', 'umbrella']):
        hashtags.append('#Outdoor')
    if any(word in title_lower for word in ['tool', 'drill', 'saw', 'hammer', 'screwdriver', 'wrench', 'ladder']):
        hashtags.append('#Tools')

    # Automotive
    if any(word in title_lower for word in ['car', 'auto', 'tire', 'oil', 'battery', 'dash cam', 'seat cover', 'floor mat', 'degreaser', 'cleaner', 'wax']):
        hashtags.append('#Automotive')

    # Pet
    if any(word in title_lower for word in ['dog', 'cat', 'pet', 'treat', 'food bowl', 'leash', 'collar', 'bed', 'toy']):
        hashtags.append('#Pet')

    # Office & School

[525 more lines in file. Use offset=230 to continue.]


Let me find the download_image function and the section where we validate and download images.

exec

grep -n "def download_image\|def validate_tweet\|# 5. Download" /home/crogers2287/openclaw-workspaces/ghc-marketing/scripts/post_deal_safe.py


348:def validate_tweet(tweet):
370:def download_image(deal_id, deal=None):
713:    # 5. Download image


Good, I found the line numbers. Let me read the download_image function and then add image hash validation.


def download_image(deal_id, deal=None):
    """Download product image from deal's image_url (NOT share endpoint)"""
    img_path = f"/tmp/deal_{deal_id}.jpg"
    try:
        # CRITICAL: Use the deal's actual image_url, NOT the share endpoint
        # The share endpoint was serving cached/wrong images
        if deal and deal.get('image_url'):
            image_url = deal['image_url']
            print(f"  Using deal image: {image_url}")
        else:
            # Fallback to share endpoint only if no deal provided (shouldn't happen)
            image_url = f"{SHARE_IMAGE_URL}{deal_id}"
            print(f"  WARNING: Using share endpoint (deal object missing)")

        req = urllib.request.Request(image_url, headers={'User-Agent': 'Mozilla/5.0'})
        with urllib.request.urlopen(req, timeout=15) as resp, open(img_path, 'wb') as f:
            f.write(resp.read())

        # Validate image isn't obviously wrong (like Apple Watch on non-watch deal)
        validation = validate_image_matches_deal(img_path, deal)
        if not validation['valid']:
            print(f"  ⚠ IMAGE VALIDATION FAILED: {validation['reason']}")
            print(f"  Skipping image for this post")
            return None

        return img_path
    except Exception as e:
        print(f"Error downloading image: {e}")
        return None

[355 more lines in file. Use offset=400 to continue.]


Interesting - there's already a `validate_image_matches_deal` function being called! But it's not defined yet in the parts I've read. Let me check if it exists.

grep -n "def validate_image_matches_deal" /home/crogers2287/openclaw-workspaces/ghc-marketing/scripts/post_deal_safe.py


400:def validate_image_matches_deal(img_path, deal):


There IS a validation function. Let me read it and see what it does.


def validate_image_matches_deal(img_path, deal):
    """Basic validation that downloaded image matches the deal"""
    if not deal:
        return {'valid': True}  # Can't validate without deal data

    title = deal.get('title', '').lower()
    image_url = deal.get('image_url', '')

    # Check if deal is about watches but image URL suggests otherwise (or vice versa)
    watch_keywords = ['watch', 'apple watch', 'smartwatch', 'fitbit', 'garmin']
    is_watch_deal = any(kw in title for kw in watch_keywords)

    # If deal is NOT about watches but image URL has suspicious patterns, flag it
    # This catches cases where the image is cached/stale
    if not is_watch_deal:
        # Check if image might be wrong based on size or other heuristics
        # Apple Watch images are typically 300x300 and have specific characteristics
        try:
            import os
            img_size = os.path.getsize(img_path)

            # Apple Watch images from Slickdeals cache are often ~6KB
            # If we have a non-watch deal with a suspiciously small image, warn
            if img_size < 7000 and 'slickdealscdn' in image_url:
                return {
                    'valid': False,
                    'reason': f"Suspicious image size ({img_size} bytes) from external URL"
                }

        except Exception:
            pass

    return {'valid': True}

def post_to_x(deal_id, tweet_text, img_path):
    """Post to X via Maya VM twikit - direct method"""
    # Copy image to Maya
    try:
        subprocess.run(['scp', '-q', img_path, f"{MAYA_HOST}:/home/maya/workspace/downloads/deal_{deal_id}.jpg"],
                      check=True, timeout=30)
    except subprocess.CalledProcessError as e:
        print(f"SCP failed: {e}")
        return None

    # Create Python script for Maya to execute
    maya_script = f'''
import asyncio
import json
from twikit import Client

[305 more lines in file. Use offset=450 to continue.]


The existing validation is very basic - it just checks image file size. This is not enough to catch the wrong image issue. I need to add:1. **Image content hash tracking** - record the hash of every image posted and detect if the same image is being used for multiple different deals
2. **Image duplication check** - if we try to post an image that was already used for a different deal, skip itLet me add a more robust validation function:

edit

def validate_image_matches_deal(img_path, deal):
    """Basic validation that downloaded image matches the deal"""
    if not deal:
        return {'valid': True}  # Can't validate without deal data

    title = deal.get('title', '').lower()
    image_url = deal.get('image_url', '')

    # Check if deal is about watches but image URL suggests otherwise (or vice versa)
    watch_keywords = ['watch', 'apple watch', 'smartwatch', 'fitbit', 'garmin']
    is_watch_deal = any(kw in title for kw in watch_keywords)

    # If deal is NOT about watches but image URL has suspicious patterns, flag it
    # This catches cases where the image is cached/stale
    if not is_watch_deal:
        # Check if image might be wrong based on size or other heuristics
        # Apple Watch images are typically 300x300 and have specific characteristics
        try:
            import os
            img_size = os.path.getsize(img_path)

            # Apple Watch images from Slickdeals cache are often ~6KB
            # If we have a non-watch deal with a suspiciously small image, warn
            if img_size < 7000 and 'slickdealscdn' in image_url:
                return {
                    'valid': False,
                    'reason': f"Suspicious image size ({img_size} bytes) from external URL"
                }

        except Exception:
            pass

    return {'valid': True}

def get_image_hash_db():
    """Load the image hash database"""
    try:
        if os.path.exists(IMAGE_HASH_DB):
            with open(IMAGE_HASH_DB, 'r') as f:
                return json.load(f)
    except Exception:
        pass
    return {}  # {image_hash: deal_id}

def save_image_hash_db(db):
    """Save the image hash database"""
    try:
        os.makedirs(os.path.dirname(IMAGE_HASH_DB), exist_ok=True)
        with open(IMAGE_HASH_DB, 'w') as f:
            json.dump(db, f, indent=2)
    except Exception as e:
        print(f"Warning: Could not save image hash db: {e}")

def compute_image_hash(img_path):
    """Compute MD5 hash of image file contents"""
    try:
        with open(img_path, 'rb') as f:
            return hashlib.md5(f.read()).hexdigest()
    except Exception:
        return None

def validate_image_matches_deal(img_path, deal):
    """Validate that downloaded image matches the deal and hasn't been used before"""
    if not deal:
        return {'valid': True, 'hash': None}  # Can't validate without deal data

    deal_id = deal.get('id')
    title = deal.get('title', '').lower()
    image_url = deal.get('image_url', '')

    # Compute image content hash
    img_hash = compute_image_hash(img_path)
    if not img_hash:
        return {'valid': True, 'hash': None, 'reason': 'Could not compute hash'}

    # Check if this image has been used for a DIFFERENT deal
    hash_db = get_image_hash_db()
    if img_hash in hash_db:
        prev_deal_id = hash_db[img_hash]
        if prev_deal_id != deal_id:
            return {
                'valid': False,
                'hash': img_hash,
                'reason': f"Image already used for deal {prev_deal_id} - DUPLICATE IMAGE DETECTED"
            }

    # Check if deal is about watches but image URL suggests otherwise (or vice versa)
    watch_keywords = ['watch', 'apple watch', 'smartwatch', 'fitbit', 'garmin']
    is_watch_deal = any(kw in title for kw in watch_keywords)

    # Size-based validation for suspicious images
    if not is_watch_deal:
        try:
            import os
            img_size = os.path.getsize(img_path)

            # Very small images (< 5KB) from external URLs are suspicious
            if img_size < 5000 and any(domain in image_url for domain in ['slickdealscdn', 'camelcamelcamel']):
                return {
                    'valid': False,
                    'hash': img_hash,
                    'reason': f"Suspicious small image ({img_size} bytes) from external URL - likely wrong image"
                }
        except Exception:
            pass

    return {'valid': True, 'hash': img_hash}

def record_image_hash(img_hash, deal_id):
    """Record that an image was used for a deal"""
    if not img_hash:
        return
    hash_db = get_image_hash_db()
    hash_db[img_hash] = deal_id
    # Keep only last 1000 entries to prevent unbounded growth
    if len(hash_db) > 1000:
        # Remove oldest entries (simple approach - just keep most recent)
        hash_db = dict(list(hash_db.items())[-1000:])
    save_image_hash_db(hash_db)


     ...
 396     except Exception as e:
 397         print(f"Error downloading image: {e}")
 398         return None
 399
+400 def get_image_hash_db():
+401     """Load the image hash database"""
+402     try:
+403         if os.path.exists(IMAGE_HASH_DB):
+404             with open(IMAGE_HASH_DB, 'r') as f:
+405                 return json.load(f)
+406     except Exception:
+407         pass
+408     return {}  # {image_hash: deal_id}
+409
+410 def save_image_hash_db(db):
+411     """Save the image hash database"""
+412     try:
+413         os.makedirs(os.path.dirname(IMAGE_HASH_DB), exist_ok=True)
+414         with open(IMAGE_HASH_DB, 'w') as f:
+415             json.dump(db, f, indent=2)
+416     except Exception as e:
+417         print(f"Warning: Could not save image hash db: {e}")
+418
+419 def compute_image_hash(img_path):
+420     """Compute MD5 hash of image file contents"""
+421     try:
+422         with open(img_path, 'rb') as f:
+423             return hashlib.md5(f.read()).hexdigest()
+424     except Exception:
+425         return None
+426
 400 def validate_image_matches_deal(img_path, deal):
-401     """Basic validation that downloaded image matches the deal"""
+428     """Validate that downloaded image matches the deal and hasn't been used before"""
 402     if not deal:
-403         return {'valid': True}  # Can't validate without deal data
+430         return {'valid': True, 'hash': None}  # Can't validate without deal data
 404
+432     deal_id = deal.get('id')
 405     title = deal.get('title', '').lower()
 406     image_url = deal.get('image_url', '')
 407
+436     # Compute image content hash
+437     img_hash = compute_image_hash(img_path)
+438     if not img_hash:
+439         return {'valid': True, 'hash': None, 'reason': 'Could not compute hash'}
+440
+441     # Check if this image has been used for a DIFFERENT deal
+442     hash_db = get_image_hash_db()
+443     if img_hash in hash_db:
+444         prev_deal_id = hash_db[img_hash]
+445         if prev_deal_id != deal_id:
+446             return {
+447                 'valid': False,
+448                 'hash': img_hash,
+449                 'reason': f"Image already used for deal {prev_deal_id} - DUPLICATE IMAGE DETECTED"
+450             }
+451
 408     # Check if deal is about watches but image URL suggests otherwise (or vice versa)
 409     watch_keywords = ['watch', 'apple watch', 'smartwatch', 'fitbit', 'garmin']
 410     is_watch_deal = any(kw in title for kw in watch_keywords)
 411
-412     # If deal is NOT about watches but image URL has suspicious patterns, flag it
-413     # This catches cases where the image is cached/stale
+456     # Size-based validation for suspicious images
 414     if not is_watch_deal:
-415         # Check if image might be wrong based on size or other heuristics
-416         # Apple Watch images are typically 300x300 and have specific characteristics
 417         try:
 418             import os
 419             img_size = os.path.getsize(img_path)
 420
-421             # Apple Watch images from Slickdeals cache are often ~6KB
-422             # If we have a non-watch deal with a suspiciously small image, warn
-423             if img_size < 7000 and 'slickdealscdn' in image_url:
+462             # Very small images (< 5KB) from external URLs are suspicious
+463             if img_size < 5000 and any(domain in image_url for domain in ['slickdealscdn', 'camelcamelcamel']):
 424                 return {
 425                     'valid': False,
-426                     'reason': f"Suspicious image size ({img_size} bytes) from external URL"
+466                     'hash': img_hash,
+467                     'reason': f"Suspicious small image ({img_size} bytes) from external URL - likely wrong image"
 427                 }
-428
 429         except Exception:
 430             pass
 431
-432     return {'valid': True}
+472     return {'valid': True, 'hash': img_hash}
 433
+474 def record_image_hash(img_hash, deal_id):
+475     """Record that an image was used for a deal"""
+476     if not img_hash:
+477         return
+478     hash_db = get_image_hash_db()
+479     hash_db[img_hash] = deal_id
+480     # Keep only last 1000 entries to prevent unbounded growth
+481     if len(hash_db) > 1000:
+482         # Remove oldest entries (simple approach - just keep most recent)
+483         hash_db = dict(list(hash_db.items())[-1000:])
+484     save_image_hash_db(hash_db)
+485
 434 def post_to_x(deal_id, tweet_text, img_path):
 435     """Post to X via Maya VM twikit - direct method"""
 436     # Copy image to Maya
 437     try:
     ...


Now I need to:
1. Update the download_image function to use the new validation response format
2. Record the image hash after successful posting

Let me update download_image to handle the new validation response:

def download_image(deal_id, deal=None):
    """Download product image from deal's image_url (NOT share endpoint)"""
    img_path = f"/tmp/deal_{deal_id}.jpg"
    try:
        # CRITICAL: Use the deal's actual image_url, NOT the share endpoint
        # The share endpoint was serving cached/wrong images
        if deal and deal.get('image_url'):
            image_url = deal['image_url']
            print(f"  Using deal image: {image_url}")
        else:
            # Fallback to share endpoint only if no deal provided (shouldn't happen)
            image_url = f"{SHARE_IMAGE_URL}{deal_id}"
            print(f"  WARNING: Using share endpoint (deal object missing)")

        req = urllib.request.Request(image_url, headers={'User-Agent': 'Mozilla/5.0'})
        with urllib.request.urlopen(req, timeout=15) as resp, open(img_path, 'wb') as f:
            f.write(resp.read())

        # Validate image isn't obviously wrong (like Apple Watch on non-watch deal)
        validation = validate_image_matches_deal(img_path, deal)
        if not validation['valid']:
            print(f"  ⚠ IMAGE VALIDATION FAILED: {validation['reason']}")
            print(f"  Skipping image for this post")
            return None

        return img_path
    except Exception as e:
        print(f"Error downloading image: {e}")
        return None

def download_image(deal_id, deal=None):
    """Download product image from deal's image_url (NOT share endpoint)"""
    img_path = f"/tmp/deal_{deal_id}.jpg"
    try:
        # CRITICAL: Use the deal's actual image_url, NOT the share endpoint
        # The share endpoint was serving cached/wrong images
        if deal and deal.get('image_url'):
            image_url = deal['image_url']
            print(f"  Using deal image: {image_url}")
        else:
            # Fallback to share endpoint only if no deal provided (shouldn't happen)
            image_url = f"{SHARE_IMAGE_URL}{deal_id}"
            print(f"  WARNING: Using share endpoint (deal object missing)")

        req = urllib.request.Request(image_url, headers={'User-Agent': 'Mozilla/5.0'})
        with urllib.request.urlopen(req, timeout=15) as resp, open(img_path, 'wb') as f:
            f.write(resp.read())

        # Validate image isn't obviously wrong (like Apple Watch on non-watch deal)
        validation = validate_image_matches_deal(img_path, deal)
        if not validation['valid']:
            print(f"  ❌ IMAGE VALIDATION FAILED: {validation['reason']}")
            print(f"  Skipping this deal to avoid posting wrong image")
            return None

        # Store hash for tracking
        img_hash = validation.get('hash')
        if img_hash:
            print(f"  ✓ Image hash: {img_hash[:12]}...")

        return img_path
    except Exception as e:
        print(f"Error downloading image: {e}")
        return None


     ...
 387
 388         # Validate image isn't obviously wrong (like Apple Watch on non-watch deal)
 389         validation = validate_image_matches_deal(img_path, deal)
 390         if not validation['valid']:
-391             print(f"  ⚠ IMAGE VALIDATION FAILED: {validation['reason']}")
-392             print(f"  Skipping image for this post")
+391             print(f"  ❌ IMAGE VALIDATION FAILED: {validation['reason']}")
+392             print(f"  Skipping this deal to avoid posting wrong image")
 393             return None
 394
+395         # Store hash for tracking
+396         img_hash = validation.get('hash')
+397         if img_hash:
+398             print(f"  ✓ Image hash: {img_hash[:12]}...")
+399
 395         return img_path
 396     except Exception as e:
 397         print(f"Error downloading image: {e}")
 398         return None
     ...


Now I need to add recording of the image hash after a successful post. Let me find where we record the post and add the hash recording there. I need to also pass the image hash from download_image through to the post_to_x function.

Actually, a simpler approach: let me record the image hash right after validation passes in download_image, since that's when we know we're going to use this image.

def download_image(deal_id, deal=None):
    """Download product image from deal's image_url (NOT share endpoint)"""
    img_path = f"/tmp/deal_{deal_id}.jpg"
    try:
        # CRITICAL: Use the deal's actual image_url, NOT the share endpoint
        # The share endpoint was serving cached/wrong images
        if deal and deal.get('image_url'):
            image_url = deal['image_url']
            print(f"  Using deal image: {image_url}")
        else:
            # Fallback to share endpoint only if no deal provided (shouldn't happen)
            image_url = f"{SHARE_IMAGE_URL}{deal_id}"
            print(f"  WARNING: Using share endpoint (deal object missing)")

        req = urllib.request.Request(image_url, headers={'User-Agent': 'Mozilla/5.0'})
        with urllib.request.urlopen(req, timeout=15) as resp, open(img_path, 'wb') as f:
            f.write(resp.read())

        # Validate image isn't obviously wrong (duplicate detection, suspicious size, etc.)
        validation = validate_image_matches_deal(img_path, deal)
        if not validation['valid']:
            print(f"  ❌ IMAGE VALIDATION FAILED: {validation['reason']}")
            print(f"  Skipping this deal to avoid posting wrong image")
            return None

        # Record image hash to prevent future duplicates
        img_hash = validation.get('hash')
        if img_hash and deal:
            record_image_hash(img_hash, deal.get('id'))
            print(f"  ✓ Image hash recorded: {img_hash[:12]}...")

        return img_path
    except Exception as e:
        print(f"Error downloading image: {e}")
        return None

def download_image(deal_id, deal=None):
    """Download product image from deal's image_url (NOT share endpoint)"""
    img_path = f"/tmp/deal_{deal_id}.jpg"
    try:
        # CRITICAL: Use the deal's actual image_url, NOT the share endpoint
        # The share endpoint was serving cached/wrong images
        if deal and deal.get('image_url'):
            image_url = deal['image_url']
            print(f"  Using deal image: {image_url}")
        else:
            # Fallback to share endpoint only if no deal provided (shouldn't happen)
            image_url = f"{SHARE_IMAGE_URL}{deal_id}"
            print(f"  WARNING: Using share endpoint (deal object missing)")

        req = urllib.request.Request(image_url, headers={'User-Agent': 'Mozilla/5.0'})
        with urllib.request.urlopen(req, timeout=15) as resp, open(img_path, 'wb') as f:
            f.write(resp.read())

        # Validate image isn't obviously wrong (like Apple Watch on non-watch deal)
        validation = validate_image_matches_deal(img_path, deal)
        if not validation['valid']:
            print(f"  ❌ IMAGE VALIDATION FAILED: {validation['reason']}")
            print(f"  Skipping this deal to avoid posting wrong image")
            return None

        # Store hash for tracking
        img_hash = validation.get('hash')
        if img_hash:
            print(f"  ✓ Image hash: {img_hash[:12]}...")

        return img_path
    except Exception as e:
        print(f"Error downloading image: {e}")
        return None


     ...
 384         req = urllib.request.Request(image_url, headers={'User-Agent': 'Mozilla/5.0'})
 385         with urllib.request.urlopen(req, timeout=15) as resp, open(img_path, 'wb') as f:
 386             f.write(resp.read())
 387
-388         # Validate image isn't obviously wrong (like Apple Watch on non-watch deal)
+388         # Validate image isn't obviously wrong (duplicate detection, suspicious size, etc.)
 389         validation = validate_image_matches_deal(img_path, deal)
 390         if not validation['valid']:
 391             print(f"  ❌ IMAGE VALIDATION FAILED: {validation['reason']}")
 392             print(f"  Skipping this deal to avoid posting wrong image")
 393             return None
 394
-395         # Store hash for tracking
+395         # Record image hash to prevent future duplicates
 396         img_hash = validation.get('hash')
-397         if img_hash:
-398             print(f"  ✓ Image hash: {img_hash[:12]}...")
+397         if img_hash and deal:
+398             record_image_hash(img_hash, deal.get('id'))
+399             print(f"  ✓ Image hash recorded: {img_hash[:12]}...")
 399
 400         return img_path
 401     except Exception as e:
 402         print(f"Error downloading image: {e}")
     ...