generated from templates/go-generic
checkpoint: download and read HTML
This commit is contained in:
parent
235132d0cc
commit
0627b4c6bb
1 changed files with 49 additions and 1 deletions
50
main.go
50
main.go
|
@ -3,7 +3,11 @@ package main
|
|||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
|
@ -33,7 +37,51 @@ func run() error {
|
|||
|
||||
baseURL := args[0]
|
||||
|
||||
fmt.Printf("starting crawl of: %s\n", baseURL)
|
||||
htmlBody, err := getHTML(baseURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Println(htmlBody)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getHTML(rawURL string) (string, error) {
|
||||
req, err := http.NewRequest(http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error creating the request: %w", err)
|
||||
}
|
||||
|
||||
client := http.Client{
|
||||
Timeout: time.Duration(10 * time.Second),
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error getting the response: %w", err)
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return "", fmt.Errorf(
|
||||
"received a bad status from %s: (%d) %s",
|
||||
rawURL,
|
||||
resp.StatusCode,
|
||||
resp.Status,
|
||||
)
|
||||
}
|
||||
|
||||
contentType := resp.Header.Get("content-type")
|
||||
if !strings.Contains(contentType, "text/html") {
|
||||
return "", fmt.Errorf("unexpected content type received: want text/html, got %s", contentType)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error reading the data from the response: %w", err)
|
||||
}
|
||||
|
||||
return string(data), nil
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue