SEO-Dashboard

SEO Dashboard using R and ScreamingFrog for Reporting and Monitoring

Paaslogs Configuration

Input section

input { beats { port => 5044 ssl => true ssl_certificate => "/etc/ssl/private/server.crt" ssl_key => "/etc/ssl/private/server.key" } }

Filter section

filter { mutate { rename => { "source" => "filename" } }

if [type] == "apache" {  
  mutate {
      add_field => { 
             "section" => "nohtml"
             "active" => "FALSE"
       }
    }
   grok {
       match => { "message" => "%{OVHCOMMONAPACHELOG}" }
       patterns_dir => "/opt/logstash/patterns"
   }
   if ("_grokparsefailure" in [tags]) {
       mutate {
          remove_tag => [ "_grokparsefailure" ]
        }
       grok {
          match => [ "message", "%{OVHCOMBINEDAPACHELOG}" ]
          patterns_dir => "/opt/logstash/patterns"
         }
    }
    elasticsearch { 
      hosts => "laas.runabove.com" 
      index => "XXyourindexXX" 
      user => "ra-logs-XXX" 
      password => "2OkHXXXXXXX"        
      ssl => true 
      query => 'type:csv AND request:"%{[request]}"'
      fields => [["section","section"],["active","active"],["speed","speed"],["compliant","compliant"],["depth","depth"],["inlinks","inlinks"],["outlinks","outlinks"],["status_title","status_title"],["status_description","status_description"],["status_h1","status_h1"],["group_inlinks","group_inlinks"],["group_wordcount","group_wordcount"]]
    }
   dns {
      action => "replace"
      reverse => [ "clientip" ]
   }
   if [clientip] =~ /googlebot.com/ { 
      mutate {
         add_field => { "bot" => "google" }
     }
   }
  if [clientip] =~ /search.msn.com/ { 
     mutate {
        add_field => { "bot" => "bing" }
     }
  }
  
  mutate {
     convert => { 
         "depth" => "integer"
         "inlinks" => "integer"
         "outlinks" => "integer"
     }
  }	  
  
}
if [type] == "csv" {
    csv {
        columns => ["request", "section","active", "speed", "compliant","depth","inlinks","outlinks","status_title","status_description","status_h1","group_inlinks","group_wordcount"]
        separator => ";"
    } 
}

}

Custom grok patterns

OVHCOMMONAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} [%{HTTPDATE:timestamp}] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion_num:float})?|%{DATA:rawrequest})" %{NUMBER:response_int:int} (?:%{NUMBER:bytes_int:int}|-)
OVHCOMBINEDAPACHELOG %{OVHCOMMONAPACHELOG} "%{NOTSPACE:referrer}" %{QS:agent}
OVHCOMMONIISLOG %{TIMESTAMP_ISO8601:log_timestamp} %{WORD:iisSite} %{IPORHOST:site} %{WORD:method} %{URIPATH:page} %{NOTSPACE:querystring} %{NUMBER:port} %{NOTSPACE:username} %{IPORHOST:clienthost} %{NOTSPACE:useragent} %{NOTSPACE:referer} %{NUMBER:response} %{NUMBER:subresponse} %{NUMBER:scstatus} %{NUMBER:bytes:int} %{NUMBER:timetaken:int}

voltek62/SEO-Dashboard

SEO-Dashboard

Paaslogs Configuration