Leads Automation: Using LLM to estimate webpage freshness

from bs4 import BeautifulSoup
import re
import csv
import requests
import openai

'''
import os
print("API Key:", os.getenv("OPENAI_API_KEY"))
exit()
'''


def estimate_last_update_with_llm(url):
    """Estimate the last update date of a webpage using ChatGPT."""
    try:
        # Fetch the page content
        response = requests.get(url, timeout=10)
        response.raise_for_status()
        page_content = response.text[:5000]  # Limit to first 5000 characters

        # Construct prompt for ChatGPT
        '''
        prompt = (
            "Based on the following webpage content, estimate the most recent update date if possible. "
            "Look for any visible timestamps, published or modified dates, or context that might indicate recency.\n\n"
            f"Content:\n{page_content}\n\nEstimated last update date:"
        )
        '''
        prompt = (
            "Based on the following URL, estimate the most recent update date if possible. "
            "Look for any visible timestamps, published or modified dates, or context that might indicate recency.\n\n"
            f"URL:\n{url}\n\nEstimated last update date:"
        )       

        # Initialize OpenAI client
        client = openai.OpenAI()

        # Call OpenAI API
        response = client.chat.completions.create(
            model="gpt-4",
            messages=[{"role": "system", "content": "You are an expert in analyzing webpage content."},
                      #{"role": "user", "content": prompt}]
                      {"role": "user", "content": f"Based on this webpage, when was it last updated?\n\n{page_content}"}
                      ])
        

            
        # Extract the estimated date from response
        estimated_date = response.choices[0].message.content.strip()
        return estimated_date

    except requests.RequestException:
        return "Error fetching the webpage"
    except Exception as e:
        return f"LLM estimation failed: {str(e)}"

def get_last_modified(url):
    """Retrieve the Last-Modified header from a URL if available."""
    try:
        response = requests.head(url, timeout=5)
        return response.headers.get("Last-Modified", "Unknown")
    except requests.RequestException:
        return "Unknown"

def extract_urls_from_html(html_path):
    """Extract all URLs, their link text, and last modified dates from an HTML file."""
    
    with open(html_path, "r", encoding="utf-8") as file:
        soup = BeautifulSoup(file, "html.parser")

    url_data = set()  # Use a set to avoid duplicates

    # Extract URLs from <a href="…"> and their link text
    count = 0
    for link in soup.find_all("a", href=True):
        url = link["href"].strip()
        link_text = link.get_text(strip=True)  # Extract visible link text
        last_modified = estimate_last_update_with_llm(url)
        url_data.add((url, link_text, last_modified))
        print(f"{count} : {url} : {link_text} : {last_modified}")
        count += 1

    # Extract URLs appearing as plain text
    url_pattern = re.compile(r"https?://[^\s\"'>]+")                            
    for text in soup.stripped_strings:
        for match in url_pattern.findall(text):
            last_modified = estimate_last_update_with_llm(match)
            url_data.add((match, "", last_modified))  # No link text for plain text URLs

    return list(url_data)

def save_urls_to_csv(url_data, output_file):
    """Save extracted URLs, their link text, and last modified dates to a CSV file."""
    with open(output_file, "w", newline="", encoding="utf-8") as file:
        writer = csv.writer(file)
        writer.writerow(["URL", "Link Text", "Last Modified"])  # Header row
        writer.writerows(url_data)
    print(f"URLs saved to {output_file}")

if __name__ == "__main__":
    html_path = "test_leads.html" #just do a small list for testing
    extracted_urls = extract_urls_from_html(html_path)

    if extracted_urls:
        output_file = "lead_urls.csv"
        print("Extracted URLs:")
        for url, text, last_modified in extracted_urls:
            print(f"URL: {url}, Link Text: {text}, Last Modified: {last_modified}")
        save_urls_to_csv(extracted_urls, output_file)
    else:
        print("No URLs found in the HTML file.")