generated from templates/go-generic
checkpoint: download and read HTML
This commit is contained in:
parent
235132d0cc
commit
0627b4c6bb
1 changed files with 49 additions and 1 deletions
50
main.go
50
main.go
|
@ -3,7 +3,11 @@ package main
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
@ -33,7 +37,51 @@ func run() error {
|
||||||
|
|
||||||
baseURL := args[0]
|
baseURL := args[0]
|
||||||
|
|
||||||
fmt.Printf("starting crawl of: %s\n", baseURL)
|
htmlBody, err := getHTML(baseURL)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println(htmlBody)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getHTML(rawURL string) (string, error) {
|
||||||
|
req, err := http.NewRequest(http.MethodGet, rawURL, nil)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("error creating the request: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
client := http.Client{
|
||||||
|
Timeout: time.Duration(10 * time.Second),
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("error getting the response: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode >= 400 {
|
||||||
|
return "", fmt.Errorf(
|
||||||
|
"received a bad status from %s: (%d) %s",
|
||||||
|
rawURL,
|
||||||
|
resp.StatusCode,
|
||||||
|
resp.Status,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
contentType := resp.Header.Get("content-type")
|
||||||
|
if !strings.Contains(contentType, "text/html") {
|
||||||
|
return "", fmt.Errorf("unexpected content type received: want text/html, got %s", contentType)
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("error reading the data from the response: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return string(data), nil
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in a new issue