ai-podcast/src/app/api/scrape/route.ts

import { NextRequest, NextResponse } from 'next/server';
import Firecrawl from '@mendable/firecrawl-js';

export async function POST(request: NextRequest) {
  try {
    const { url } = await request.json();

    if (!url) {
      return NextResponse.json(
        { error: 'URL is required' },
        { status: 400 }
      );
    }

    // Initialize Firecrawl with API key from environment
    const firecrawl = new Firecrawl({ 
      apiKey: process.env.FIRECRAWL_API_KEY 
    });

    console.log('Attempting to scrape URL:', url);

    // Scrape the website
    const result = await firecrawl.scrape(url, {
      formats: ['markdown', 'html']
    });

    console.log('Firecrawl result received');

    // Check if we have the expected data structure
    if (!result || !result.markdown) {
      console.error('Invalid Firecrawl response:', result);
      throw new Error('Invalid response from Firecrawl API');
    }

    // Create a truncated excerpt for display
    const excerpt = result.markdown 
      ? result.markdown.substring(0, 200) + (result.markdown.length > 200 ? '...' : '')
      : 'No content available';

    return NextResponse.json({
      success: true,
      data: {
        url: result.metadata?.sourceURL || url,
        title: result.metadata?.title || 'Untitled',
        description: result.metadata?.description || '',
        content: result.markdown || '',
        excerpt: excerpt,
        scrapedAt: new Date().toISOString()
      }
    });

  } catch (error) {
    console.error('Scraping error:', error);
    return NextResponse.json(
      { error: error instanceof Error ? error.message : 'Failed to scrape website' },
      { status: 500 }
    );
  }
}