Unable to send any logs to splunk enterprise 8.1.9 using OTEL
bala151187 opened this issue · 2 comments
bala151187 commented
Describe the issue you're reporting
We are currently migrating from splunk connect for kubernetes to Splunk OpenTelemetry Collector for Kubernetes using helm and after installing OTEL we don't see any logs in splunk dashboard. below is the OTEL yaml . can you please let us know wat we are doing wrong
exporters:
splunk_hec/platform_logs:
disable_compression: true
endpoint: <REDACTED>
idle_conn_timeout: 10s
index: <REDACTED>
max_connections: 200
profiling_data_enabled: false
retry_on_failure:
enabled: true
initial_interval: 5s
max_elapsed_time: 300s
max_interval: 30s
sending_queue:
enabled: true
num_consumers: 10
queue_size: 5000
source: kubernetes
splunk_app_name: splunk-otel-collector
splunk_app_version: 0.81.0
timeout: 10s
tls:
insecure_skip_verify: false
token: ${SPLUNK_PLATFORM_HEC_TOKEN}
extensions:
file_storage:
directory: /var/addon/splunk/otel_pos
health_check: null
k8s_observer:
auth_type: serviceAccount
node: ${K8S_NODE_NAME}
memory_ballast:
size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
zpages: null
processors:
batch: null
filter/logs:
logs:
exclude:
match_type: strict
resource_attributes:
- key: splunk.com/exclude
value: "true"
k8sattributes:
extract:
annotations:
- from: pod
key: splunk.com/sourcetype
- from: namespace
key: splunk.com/exclude
tag_name: splunk.com/exclude
- from: pod
key: splunk.com/exclude
tag_name: splunk.com/exclude
- from: namespace
key: splunk.com/index
tag_name: com.splunk.index
- from: pod
key: splunk.com/index
tag_name: com.splunk.index
labels:
- key: app
metadata:
- k8s.namespace.name
- k8s.node.name
- k8s.pod.name
- k8s.pod.uid
- container.id
- container.image.name
- container.image.tag
filter:
node_from_env_var: K8S_NODE_NAME
pod_association:
- sources:
- from: resource_attribute
name: k8s.pod.uid
- sources:
- from: resource_attribute
name: k8s.pod.ip
- sources:
- from: resource_attribute
name: ip
- sources:
- from: connection
- sources:
- from: resource_attribute
name: host.name
memory_limiter:
check_interval: 2s
limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}
resource:
attributes:
- action: insert
key: k8s.node.name
value: ${K8S_NODE_NAME}
- action: upsert
key: k8s.cluster.name
value: contain-eks-lab
resource/add_agent_k8s:
attributes:
- action: insert
key: k8s.pod.name
value: ${K8S_POD_NAME}
- action: insert
key: k8s.pod.uid
value: ${K8S_POD_UID}
- action: insert
key: k8s.namespace.name
value: ${K8S_NAMESPACE}
resource/logs:
attributes:
- action: upsert
from_attribute: k8s.pod.annotations.splunk.com/sourcetype
key: com.splunk.sourcetype
- action: delete
key: k8s.pod.annotations.splunk.com/sourcetype
- action: delete
key: splunk.com/exclude
resourcedetection:
detectors:
- env
- eks
- system
override: true
timeout: 10s
receivers:
filelog:
encoding: utf-8
exclude:
- /var/log/pods/splunk_splunk-app-splunk-otel-collector*_*/otel-collector/*.log
fingerprint_size: 1kb
force_flush_period: "0"
include:
- /var/log/pods/*/*/*.log
include_file_name: false
include_file_path: true
max_concurrent_files: 1024
max_log_size: 1MiB
operators:
- id: parser-containerd
regex: ^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) ?(?P<log>.*)$
timestamp:
layout: '%Y-%m-%dT%H:%M:%S.%LZ'
parse_from: attributes.time
type: regex_parser
- combine_field: attributes.log
combine_with: ""
id: containerd-recombine
is_last_entry: attributes.logtag == 'F'
max_log_size: 1048576
output: handle_empty_log
source_identifier: attributes["log.file.path"]
type: recombine
- field: attributes.log
id: handle_empty_log
if: attributes.log == nil
type: add
value: ""
- parse_from: attributes["log.file.path"]
regex: ^\/var\/log\/pods\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[^\/]+)\/(?P<container_name>[^\._]+)\/(?P<restart_count>\d+)\.log$
type: regex_parser
- from: attributes.uid
to: resource["k8s.pod.uid"]
type: move
- from: attributes.restart_count
to: resource["k8s.container.restart_count"]
type: move
- from: attributes.container_name
to: resource["k8s.container.name"]
type: move
- from: attributes.namespace
to: resource["k8s.namespace.name"]
type: move
- from: attributes.pod_name
to: resource["k8s.pod.name"]
type: move
- field: resource["com.splunk.sourcetype"]
type: add
value: EXPR("kube:container:"+resource["k8s.container.name"])
- from: attributes.stream
to: attributes["log.iostream"]
type: move
- from: attributes["log.file.path"]
to: resource["com.splunk.source"]
type: move
- from: attributes.log
id: clean-up-log-record
to: body
type: move
poll_interval: 200ms
retry_on_failure:
enabled: true
start_at: beginning
storage: file_storage
fluentforward:
endpoint: 0.0.0.0:8006
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
prometheus/agent:
config:
scrape_configs:
- job_name: otel-agent
scrape_interval: 10s
static_configs:
- targets:
- ${K8S_POD_IP}:8889
service:
extensions:
- file_storage
- health_check
- k8s_observer
- memory_ballast
- zpages
pipelines:
logs:
exporters:
- splunk_hec/platform_logs
processors:
- memory_limiter
- k8sattributes
- filter/logs
- batch
- resourcedetection
- resource
- resource/logs
receivers:
- filelog
- fluentforward
- otlp
telemetry:
logs:
level: debug
metrics:
address: 0.0.0.0:8889
omrozowicz-splunk commented
Hey, what are the agent's logs? Do you see any errors?
atoulme commented
Closing as inactive. @bala151187 please open a support case to receive further guidance. Thanks!