Scraper is a simple library to scrape and monitor multiple urls and provide prometheus metrics.
package main
import (
"fmt"
"log"
"net/http"
"net/url"
"os"
"strings"
"time"
"github.com/arriqaaq/scraper"
"github.com/gorilla/mux"
)
var (
// server settings
port = 8080
readTimeout = 5 * time.Second
writeTimeout = 5 * time.Second
// targets to scrape
targets []*scraper.Target
// scraper settings
storeSize = 10
)
// healthzHandler for reporting health
func healthzHandler(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodGet:
w.WriteHeader(http.StatusOK)
w.Write([]byte("OK"))
break
default:
w.WriteHeader(http.StatusMethodNotAllowed)
}
}
func parseURLs(urls []string) []*scraper.Target {
var targets []*scraper.Target
for _, u := range urls {
u, err := url.ParseRequestURI(u)
if err != nil {
continue
}
targets = append(targets, scraper.NewTarget(u))
}
return targets
}
func main() {
//Read URLs to monitor from environment variable
var urls = []string{"http://google.com", "http://cloudflare.com", "http://reddit.com"}
targets = parseURLs(urls)
fmt.Printf("URLs to monitor: %+v\n", targets)
scrapePool, err := scraper.NewScrapePool(
&scraper.ScrapeConfig{
ScrapeInterval: time.Duration(3 * time.Second),
ScrapeTimeout: time.Duration(2 * time.Second),
StoreSize: storeSize,
},
)
if err != nil {
panic(err)
}
scraper.RegisterExporter(scrapePool)
// start scraping the targets
scrapePool.Start(targets)
// create Router
router := mux.NewRouter()
// register handlers
router.Handle("/metrics", scraper.PrometheusHandler())
router.HandleFunc("/healthz", healthzHandler)
// configure the HTTP server and start it
s := &http.Server{
Addr: fmt.Sprintf(":%d", port),
ReadTimeout: readTimeout,
WriteTimeout: writeTimeout,
MaxHeaderBytes: http.DefaultMaxHeaderBytes,
Handler: router,
}
log.Fatal(s.ListenAndServe())
}
