gateway/test_web_csv_functionality.py

#!/usr/bin/env python3
"""
Test script for web CSV functionality
Tests both CSV output generation and CSV input reading
"""

import sys
import os
import asyncio
from typing import Dict, Any

# Add the gateway directory to the Python path
sys.path.insert(0, os.path.join(os.path.dirname(__file__)))

from modules.methods.methodWeb import MethodWeb
from modules.interfaces.interfaceWebModel import WebSearchResultItem, WebSearchDocumentData, WebSearchActionDocument, WebSearchActionResult
from pydantic import HttpUrl


def create_mock_web_search_result():
    """Create a mock WebSearchActionResult with the provided example data"""
    
    # Create mock search result items based on the provided example
    results = [
        WebSearchResultItem(
            title="Switzerland Market Analysis :: Fitch Solutions",
            url=HttpUrl("https://www.fitchsolutions.com/bmi/region/switzerland")
        ),
        WebSearchResultItem(
            title="OECD Economic Outlook, Volume 2024 Issue 2: Switzerland",
            url=HttpUrl("https://www.oecd.org/en/publications/2024/12/oecd-economic-outlook-volume-2024-issue-2_67bb8fac/full-report/switzerland_605fd31f.html")
        ),
        WebSearchResultItem(
            title="The economic context of Switzerland - International Trade Portal",
            url=HttpUrl("https://www.lloydsbanktrade.com/en/market-potential/switzerland/economical-context")
        ),
        WebSearchResultItem(
            title="Switzerland: Country File, Economic Risk Analysis | Coface",
            url=HttpUrl("https://www.coface.com/news-economy-and-insights/business-risk-dashboard/country-risk-files/switzerland")
        ),
        WebSearchResultItem(
            title="Swiss Economic Outlook 2025 - Roland Berger",
            url=HttpUrl("https://www.rolandberger.com/en/Insights/Publications/Swiss-Economic-Outlook-2025.html")
        )
    ]
    
    # Create document data
    document_data = WebSearchDocumentData(
        query="current market trends Switzerland business economy 2024 analysis report",
        results=results,
        total_count=len(results)
    )
    
    # Create action document
    action_document = WebSearchActionDocument(
        documentName="test_search_results.json",
        documentData=document_data,
        mimeType="application/json"
    )
    
    # Create action result
    action_result = WebSearchActionResult(
        success=True,
        documents=[action_document]
    )
    
    return action_result


def test_csv_output_generation():
    """Test CSV output generation from web search results"""
    print("Testing CSV output generation...")
    
    # Create method instance (without service center for testing)
    method = MethodWeb(None)
    
    # Create mock search result
    mock_result = create_mock_web_search_result()
    
    # Convert to CSV
    csv_content = method._convert_search_results_to_csv(mock_result)
    
    print("Generated CSV content:")
    print(csv_content)
    print()
    
    # Verify CSV format
    lines = csv_content.strip().split('\n')
    assert len(lines) == 6, f"Expected 6 lines (header + 5 results), got {len(lines)}"
    
    # Check header
    assert lines[0] == "url;title", f"Expected header 'url;title', got '{lines[0]}'"
    
    # Check that URLs are present
    for i, line in enumerate(lines[1:], 1):
        parts = line.split(';')
        assert len(parts) == 2, f"Line {i} should have 2 parts separated by ';', got {len(parts)}"
        url, title = parts
        assert url.startswith('https://'), f"Line {i} URL should start with 'https://', got '{url}'"
        assert title, f"Line {i} should have a title, got empty title"
    
    print("✓ CSV output generation test passed!")
    return csv_content


def test_csv_input_reading():
    """Test CSV input reading functionality"""
    print("Testing CSV input reading...")
    
    # Create method instance
    method = MethodWeb(None)
    
    # Test semicolon-separated CSV
    semicolon_csv = """url;title
https://www.fitchsolutions.com/bmi/region/switzerland;Switzerland Market Analysis :: Fitch Solutions
https://www.oecd.org/en/publications/2024/12/oecd-economic-outlook-volume-2024-issue-2_67bb8fac/full-report/switzerland_605fd31f.html;OECD Economic Outlook, Volume 2024 Issue 2: Switzerland
https://www.lloydsbanktrade.com/en/market-potential/switzerland/economical-context;The economic context of Switzerland - International Trade Portal"""
    
    urls_semicolon = method._read_csv_with_urls(semicolon_csv)
    print(f"Extracted {len(urls_semicolon)} URLs from semicolon CSV:")
    for url in urls_semicolon:
        print(f"  - {url}")
    
    assert len(urls_semicolon) == 3, f"Expected 3 URLs, got {len(urls_semicolon)}"
    assert all(url.startswith('https://') for url in urls_semicolon), "All URLs should start with https://"
    
    print("✓ Semicolon CSV reading test passed!")
    
    # Test comma-separated CSV
    comma_csv = """url,title
https://www.fitchsolutions.com/bmi/region/switzerland,Switzerland Market Analysis :: Fitch Solutions
https://www.oecd.org/en/publications/2024/12/oecd-economic-outlook-volume-2024-issue-2_67bb8fac/full-report/switzerland_605fd31f.html,OECD Economic Outlook, Volume 2024 Issue 2: Switzerland"""
    
    urls_comma = method._read_csv_with_urls(comma_csv)
    print(f"Extracted {len(urls_comma)} URLs from comma CSV:")
    for url in urls_comma:
        print(f"  - {url}")
    
    assert len(urls_comma) == 2, f"Expected 2 URLs, got {len(urls_comma)}"
    assert all(url.startswith('https://') for url in urls_comma), "All URLs should start with https://"
    
    print("✓ Comma CSV reading test passed!")
    
    # Test case-insensitive column names
    case_insensitive_csv = """URL;Title
https://example.com/test;Test Title"""
    
    urls_case = method._read_csv_with_urls(case_insensitive_csv)
    assert len(urls_case) == 1, f"Expected 1 URL, got {len(urls_case)}"
    assert urls_case[0] == "https://example.com/test", f"Expected 'https://example.com/test', got '{urls_case[0]}'"
    
    print("✓ Case-insensitive CSV reading test passed!")


def test_integration():
    """Test the complete integration: generate CSV and then read it back"""
    print("Testing integration: generate CSV and read it back...")
    
    method = MethodWeb(None)
    
    # Generate CSV from mock data
    mock_result = create_mock_web_search_result()
    csv_content = method._convert_search_results_to_csv(mock_result)
    
    # Read URLs back from the generated CSV
    extracted_urls = method._read_csv_with_urls(csv_content)
    
    print(f"Generated CSV with {len(mock_result.documents[0].documentData.results)} results")
    print(f"Extracted {len(extracted_urls)} URLs from generated CSV")
    
    # Verify we got the same number of URLs
    assert len(extracted_urls) == len(mock_result.documents[0].documentData.results), \
        f"Expected {len(mock_result.documents[0].documentData.results)} URLs, got {len(extracted_urls)}"
    
    # Verify URLs match
    original_urls = [str(result.url) for result in mock_result.documents[0].documentData.results]
    for i, (original, extracted) in enumerate(zip(original_urls, extracted_urls)):
        assert original == extracted, f"URL {i} mismatch: expected '{original}', got '{extracted}'"
    
    print("✓ Integration test passed!")


if __name__ == "__main__":
    print("Running Web CSV Functionality Tests")
    print("=" * 50)
    
    try:
        # Test CSV output generation
        csv_content = test_csv_output_generation()
        print()
        
        # Test CSV input reading
        test_csv_input_reading()
        print()
        
        # Test integration
        test_integration()
        print()
        
        print("=" * 50)
        print("🎉 All tests passed successfully!")
        
    except Exception as e:
        print(f"❌ Test failed: {e}")
        import traceback
        traceback.print_exc()
        sys.exit(1)