generated from templates/go-generic
Dan Anglin
4519de764e
All checks were successful
Tests / test (pull_request) Successful in 13s
Add the source code for the web crawler. The web crawler is a simple Go CLI application that traverses through a website and generates a report of all the internal links found in the site.
50 lines
943 B
Go
50 lines
943 B
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"strconv"
|
|
|
|
"codeflow.dananglin.me.uk/apollo/web-crawler/internal/crawler"
|
|
)
|
|
|
|
func main() {
|
|
if err := run(); err != nil {
|
|
os.Stderr.WriteString("ERROR: " + err.Error() + "\n")
|
|
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
func run() error {
|
|
args := os.Args[1:]
|
|
|
|
if len(args) != 3 {
|
|
return fmt.Errorf("unexpected number of arguments received: want 3, got %d", len(args))
|
|
}
|
|
|
|
baseURL := args[0]
|
|
|
|
maxConcurrency, err := strconv.Atoi(args[1])
|
|
if err != nil {
|
|
return fmt.Errorf("unable to convert the max concurrency (%s) to an integer: %w", args[1], err)
|
|
}
|
|
|
|
maxPages, err := strconv.Atoi(args[2])
|
|
if err != nil {
|
|
return fmt.Errorf("unable to convert the max pages (%s) to an integer: %w", args[2], err)
|
|
}
|
|
|
|
c, err := crawler.NewCrawler(baseURL, maxConcurrency, maxPages)
|
|
if err != nil {
|
|
return fmt.Errorf("unable to create the crawler: %w", err)
|
|
}
|
|
|
|
go c.Crawl(baseURL)
|
|
|
|
c.Wait()
|
|
|
|
c.PrintReport()
|
|
|
|
return nil
|
|
}
|