Logs from some of the lambda functions are not flowing through
priyam-maheshwari opened this issue · 0 comments
priyam-maheshwari commented
I have the following configuration
input {
cloudwatch_logs {
log_group => ["/aws/lambda/a","/aws/lambda/b","/aws/lambda/c","/aws/lambda/d","/aws/lambda/e","/aws/lambda/f"]
start_position => "end"
access_key_id => "<access_key>"
secret_access_key => "<secret_access_key>"
region => "eu-west-2"
tags => ["cloudwatch_syslog"]
}
}
filter {
if "cloudwatch_syslog" in [tags] {
grok {
patterns_dir => ["/etc/logstash/patterns"]
match => { "message" => ["%{TIMESTAMP_ISO8601:timestampcw} > %{GREEDYDATA:message}","%{TIMESTAMP_ISO8601:timestampcw} %{GREEDYDATA:message}","%{GREEDYDATA:message}"] }
overwrite => ["message"]
remove_field => ["cloudwatch_logs","timestampcw"]
}
json {
skip_on_invalid_json => true
source => "message"
target => "data"
remove_field => ["message"]
}
if [data][type] != "report" {
drop { }
}
else {
mutate {
replace => { "app" => "my-app" }
}
}
}
}
output {
if ![data] {
elasticsearch {
hosts => "<host>"
user => "<un>"
password => "<pass>"
ilm_rollover_alias => "log_raw"
ilm_pattern => "000001"
ilm_policy => "log_raw"
}
}
else {
elasticsearch {
hosts => "<host>"
user => "<un>"
password => "<pass>"
ilm_rollover_alias => "log"
ilm_pattern => "000001"
ilm_policy => "log"
}
}
}
I can see the logs flowing through from lambda a,b,c and d, but no logs are seen from lambda e and f in Kibana.I have tried deleting sincedb file as well, but even after that logs are not seen. lambda e and f were added to config yesterday, whereas others have been there for quite sometime now