Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions mindsdb/integrations/handlers/web_handler/web_handler.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
import ipaddress
import socket
from typing import List
from urllib.parse import urlparse

import pandas as pd
from mindsdb.integrations.libs.response import HandlerStatusResponse
Expand All @@ -10,6 +13,42 @@
from mindsdb.integrations.utilities.sql_utils import FilterCondition, FilterOperator


def _validate_no_private_ip(url: str) -> None:
"""
Validates that the given URL does not resolve to a loopback, private,
link-local, multicast, or reserved IP address to prevent SSRF.
"""
parsed = urlparse(url)
hostname = parsed.hostname
if not hostname:
raise ValueError(f"Invalid URL provided: {url}")

try:
addr_info = socket.getaddrinfo(hostname, None)
except socket.gaierror as e:
raise ValueError(f"Could not resolve hostname for URL '{url}': {e}")

for info in addr_info:
ip_str = info[4][0]
try:
ip_obj = ipaddress.ip_address(ip_str)
if (
ip_obj.is_private
or ip_obj.is_loopback
or ip_obj.is_link_local
or ip_obj.is_reserved
or ip_obj.is_multicast
or ip_obj.is_unspecified
):
raise ValueError(
f"Access to private, loopback, or reserved IP addresses is forbidden: {url} ({ip_str})"
)
except ValueError as ve:
if "forbidden" in str(ve):
raise
continue


class CrawlerTable(APIResource):
def list(self, conditions: List[FilterCondition] = None, limit: int = None, **kwargs) -> pd.DataFrame:
"""
Expand Down Expand Up @@ -47,6 +86,9 @@ def list(self, conditions: List[FilterCondition] = None, limit: int = None, **kw
'You must specify what url you want to crawl, for example: SELECT * FROM web.crawler WHERE url = "someurl"'
)

for url in urls:
_validate_no_private_ip(url)

allowed_urls = config.get("web_crawling_allowed_sites", [])
if allowed_urls and not validate_urls(urls, allowed_urls):
raise ValueError(
Expand Down Expand Up @@ -96,3 +138,4 @@ def check_connection(self) -> HandlerStatusResponse:
"""
response = HandlerStatusResponse(True)
return response

Loading