<?php

/*
====================================================
SETTINGS
====================================================
*/

$startUrl = "https://ai-library.ru";

$parsedStart = parse_url($startUrl);
$domain = $parsedStart["host"] ?? "";

$allowedExt = array("html", "htm", "shtml", "php");

$visited = array();
$queue   = array($startUrl);


/*
====================================================
HTTP GET
====================================================
*/
function getHtml(string $url): string
{
    $ch = curl_init($url);
    curl_setopt_array($ch, array(
        CURLOPT_RETURNTRANSFER => true,
        CURLOPT_FOLLOWLOCATION => true,
        CURLOPT_USERAGENT => "PHP81-SitemapCrawler-NoDOM/1.0",
        CURLOPT_TIMEOUT => 15,
        CURLOPT_SSL_VERIFYPEER => false,
        CURLOPT_SSL_VERIFYHOST => false
    ));

    $html = curl_exec($ch);
    if (!is_string($html)) {
        $html = "";
    }

    curl_close($ch);
    return $html;
}


/*
====================================================
Clean path
====================================================
*/
function cleanPath(string $url): string|false
{
    $parts = parse_url($url);
    if (!isset($parts["scheme"], $parts["host"])) return false;

    $scheme = $parts["scheme"];
    $host   = $parts["host"];
    $path   = $parts["path"] ?? "/";

    $path = preg_replace("#/+#", "/", $path);

    $segments = explode("/", $path);
    $resolved = array();

    foreach ($segments as $seg) {

        if ($seg === "" || $seg === ".") continue;

        if ($seg === "..") {
            if (!empty($resolved)) array_pop($resolved);
            continue;
        }

        $resolved[] = $seg;
    }

    $newPath = "/" . implode("/", $resolved);

    if (!preg_match("#\\.[a-z0-9]+$#i", $newPath)) {
        $newPath = rtrim($newPath, "/") . "/";
    }

    return $scheme . "://" . $host . $newPath;
}


/*
====================================================
Normalize URL
====================================================
*/
function normalizeUrl(string $base, string $url, string $domain, string $startUrl): string|false
{
    // remove spaces + tabs etc
    $url = trim($url, " \t\n\r\0\x0B");

    if ($url === "" || $url === "#" || str_starts_with($url, "javascript:")) {
        return false;
    }

    // remove fragment
    $url = preg_replace("/#.*$/", "", $url);

    $startParsed = parse_url($startUrl);
    $protocol = $startParsed["scheme"] ?? "https";

    // absolute URL
    if (preg_match("#^https?://#i", $url)) {
        $p = parse_url($url);
        if (!isset($p["host"]) || $p["host"] !== $domain) return false;
        $path = $p["path"] ?? "/";
        return cleanPath($protocol . "://" . $domain . $path);
    }

    // relative URL
    $bp = parse_url($base);
    $basePath = $bp["path"] ?? "/";

    if (!str_ends_with($basePath, "/")) {
        $baseDir = dirname($basePath);
    } else {
        $baseDir = rtrim($basePath, "/");
    }

    if (str_starts_with($url, "/")) {
        $full = $protocol . "://" . $domain . $url;
    } else {
        $full = $protocol . "://" . $domain . $baseDir . "/" . $url;
    }

    return cleanPath($full);
}


/*
====================================================
Extract links
- with quotes
- without quotes
- trims spaces and tabs
====================================================
*/
function extractLinks(string $html): array
{
    $results = array();
    if ($html === "") return $results;

    preg_match_all(
        '#href\s*=\s*(?:(["\'])(.*?)\1|([^\s>]+))#i',
        $html,
        $matches,
        PREG_SET_ORDER
    );

    foreach ($matches as $m) {

        if (!empty($m[2])) {
            $href = trim($m[2], " \t\n\r\0\x0B");
        } elseif (!empty($m[3])) {
            $href = trim($m[3], " \t\n\r\0\x0B");
        } else {
            continue;
        }

        if ($href !== "") {
            $results[] = $href;
        }
    }

    return $results;
}


/*
====================================================
Check if URL is indexable
====================================================
*/
function isPage(string $url, array $allowedExt): bool
{
    $path = parse_url($url, PHP_URL_PATH);
    if (!is_string($path)) return false;

    if (!preg_match("#\\.[a-z0-9]+$#i", $path)) {
        return true;
    }

    $ext = strtolower(pathinfo($path, PATHINFO_EXTENSION));
    return in_array($ext, $allowedExt, true);
}


/*
====================================================
Crawler
====================================================
*/

echo "Starting scan...\n\n";

while (!empty($queue)) {

    $url = array_shift($queue);

    if (isset($visited[$url])) continue;
    $visited[$url] = true;

    echo $url . "\n";

    $html = getHtml($url);
    $links = extractLinks($html);

    foreach ($links as $lnk) {

        $full = normalizeUrl($url, $lnk, $domain, $startUrl);
        if (!$full) continue;

        if (!isPage($full, $allowedExt)) continue;

        if (!isset($visited[$full])) {
            $queue[] = $full;
        }
    }
}

echo "\nTotal pages: " . count($visited) . "\n";


/*
====================================================
Build sitemap.xml
====================================================
*/

echo "Saving sitemap.xml...\n";

$xml  = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
$xml .= "<urlset xmlns=\"https://www.sitemaps.org/schemas/sitemap/0.9\">\n";

foreach (array_keys($visited) as $page) {

    if (!is_string($page)) continue;

    $page = trim($page, " \t\n\r\0\x0B");
    if ($page === "") continue;

    if (!str_starts_with($page, parse_url($startUrl, PHP_URL_SCHEME) . "://")) {
        continue;
    }

    $loc = htmlspecialchars($page, ENT_QUOTES);
    if ($loc === "") continue;

    $xml .= "  <url><loc>" . $loc . "</loc></url>\n";
}

$xml .= "</urlset>\n";

file_put_contents(__DIR__ . "/sitemap.xml", $xml);

echo "Done.\n";