web-crawler/url.go

32 lines
752 B
Go

package main
import (
"fmt"
"net/url"
"strings"
)
func normaliseURL(rawURL string) (string, error) {
const normalisedFormat string = "%s%s"
parsedURL, err := url.Parse(rawURL)
if err != nil {
return "", fmt.Errorf("error parsing the URL %q: %w", rawURL, err)
}
return fmt.Sprintf(normalisedFormat, parsedURL.Hostname(), strings.TrimSuffix(parsedURL.Path, "/")), nil
}
func equalDomains(urlA, urlB string) (bool, error) {
parsedURLA, err := url.Parse(urlA)
if err != nil {
return false, fmt.Errorf("error parsing the URL %q: %w", urlA, err)
}
parsedURLB, err := url.Parse(urlB)
if err != nil {
return false, fmt.Errorf("error parsing the URL %q: %w", urlB, err)
}
return parsedURLA.Hostname() == parsedURLB.Hostname(), nil
}