Terraform module which creates AWS MSK (Managed Streaming for Kafka) resources.
See examples directory for working examples to reference:
module "msk_kafka_cluster" {
source = "terraform-aws-modules/msk-kafka-cluster/aws"
name = local.name
kafka_version = "3.5.1"
number_of_broker_nodes = 3
enhanced_monitoring = "PER_TOPIC_PER_PARTITION"
broker_node_client_subnets = ["subnet-12345678", "subnet-024681012", "subnet-87654321"]
broker_node_storage_info = {
ebs_storage_info = { volume_size = 100 }
}
broker_node_instance_type = "kafka.t3.small"
broker_node_security_groups = ["sg-12345678"]
encryption_in_transit_client_broker = "TLS"
encryption_in_transit_in_cluster = true
configuration_name = "example-configuration"
configuration_description = "Example configuration"
configuration_server_properties = {
"auto.create.topics.enable" = true
"delete.topic.enable" = true
}
jmx_exporter_enabled = true
node_exporter_enabled = true
cloudwatch_logs_enabled = true
s3_logs_enabled = true
s3_logs_bucket = "aws-msk-kafka-cluster-logs"
s3_logs_prefix = local.name
scaling_max_capacity = 512
scaling_target_value = 80
client_authentication = {
sasl = { scram = true }
}
create_scram_secret_association = true
scram_secret_association_secret_arn_list = [
aws_secretsmanager_secret.one.arn,
aws_secretsmanager_secret.two.arn,
]
# AWS Glue Registry
schema_registries = {
team_a = {
name = "team_a"
description = "Schema registry for Team A"
}
team_b = {
name = "team_b"
description = "Schema registry for Team B"
}
}
# AWS Glue Schemas
schemas = {
team_a_tweets = {
schema_registry_name = "team_a"
schema_name = "tweets"
description = "Schema that contains all the tweets"
compatibility = "FORWARD"
schema_definition = "{\"type\": \"record\", \"name\": \"r1\", \"fields\": [ {\"name\": \"f1\", \"type\": \"int\"}, {\"name\": \"f2\", \"type\": \"string\"} ]}"
tags = { Team = "Team A" }
}
team_b_records = {
schema_registry_name = "team_b"
schema_name = "records"
description = "Schema that contains all the records"
compatibility = "FORWARD"
team_b_records = {
schema_registry_name = "team_b"
schema_name = "records"
description = "Schema that contains all the records"
compatibility = "FORWARD"
schema_definition = jsonencode({
type = "record"
name = "r1"
fields = [
{
name = "f1"
type = "int"
},
{
name = "f2"
type = "string"
},
{
name = "f3"
type = "boolean"
}
]
})
tags = { Team = "Team B" }
}
}
}
tags = {
Terraform = "true"
Environment = "dev"
}
}Examples codified under the examples are intended to give users references for how to use the module(s) as well as testing/validating changes to the source code of the module. If contributing to the project, please be sure to make any appropriate updates to the relevant examples to allow maintainers to test your changes and to keep the examples up to date for users. Thank you!
| Name | Version |
|---|---|
| terraform | >= 1.0 |
| aws | >= 5.30 |
| random | >= 3.6 |
| Name | Version |
|---|---|
| aws | >= 5.30 |
| random | >= 3.6 |
No modules.
| Name | Type |
|---|---|
| aws_appautoscaling_policy.this | resource |
| aws_appautoscaling_target.this | resource |
| aws_cloudwatch_log_group.this | resource |
| aws_glue_registry.this | resource |
| aws_glue_schema.this | resource |
| aws_msk_cluster.this | resource |
| aws_msk_cluster_policy.this | resource |
| aws_msk_configuration.this | resource |
| aws_msk_scram_secret_association.this | resource |
| aws_msk_vpc_connection.this | resource |
| aws_mskconnect_custom_plugin.this | resource |
| aws_mskconnect_worker_configuration.this | resource |
| random_id.this | resource |
| aws_iam_policy_document.this | data source |
| Name | Description | Type | Default | Required |
|---|---|---|---|---|
| broker_node_az_distribution | The distribution of broker nodes across availability zones (documentation). Currently the only valid value is DEFAULT |
string |
null |
no |
| broker_node_client_subnets | A list of subnets to connect to in client VPC (documentation) | list(string) |
[] |
no |
| broker_node_connectivity_info | Information about the cluster access configuration | any |
{} |
no |
| broker_node_instance_type | Specify the instance type to use for the kafka brokers. e.g. kafka.m5.large. (Pricing info) | string |
null |
no |
| broker_node_security_groups | A list of the security groups to associate with the elastic network interfaces to control who can communicate with the cluster | list(string) |
[] |
no |
| broker_node_storage_info | A block that contains information about storage volumes attached to MSK broker nodes | any |
{} |
no |
| client_authentication | Configuration block for specifying a client authentication | any |
{} |
no |
| cloudwatch_log_group_class | Specifies the log class of the log group. Possible values are: STANDARD or INFREQUENT_ACCESS. | string |
null |
no |
| cloudwatch_log_group_kms_key_id | The ARN of the KMS Key to use when encrypting log data | string |
null |
no |
| cloudwatch_log_group_name | Name of the Cloudwatch Log Group to deliver logs to | string |
null |
no |
| cloudwatch_log_group_retention_in_days | Specifies the number of days you want to retain log events in the log group | number |
0 |
no |
| cloudwatch_logs_enabled | Indicates whether you want to enable or disable streaming broker logs to Cloudwatch Logs | bool |
false |
no |
| cluster_override_policy_documents | Override policy documents for cluster policy | list(string) |
null |
no |
| cluster_policy_statements | Map of policy statements for cluster policy | any |
null |
no |
| cluster_source_policy_documents | Source policy documents for cluster policy | list(string) |
null |
no |
| configuration_arn | ARN of an externally created configuration to use | string |
null |
no |
| configuration_description | Description of the configuration | string |
null |
no |
| configuration_name | Name of the configuration | string |
null |
no |
| configuration_revision | Revision of the externally created configuration to use | number |
null |
no |
| configuration_server_properties | Contents of the server.properties file. Supported properties are documented in the MSK Developer Guide | map(string) |
{} |
no |
| connect_custom_plugin_timeouts | Timeout configurations for the connect custom plugins | map(string) |
{ |
no |
| connect_custom_plugins | Map of custom plugin configuration details (map of maps) | any |
{} |
no |
| connect_worker_config_description | A summary description of the worker configuration | string |
null |
no |
| connect_worker_config_name | The name of the worker configuration | string |
null |
no |
| connect_worker_config_properties_file_content | Contents of connect-distributed.properties file. The value can be either base64 encoded or in raw format | string |
null |
no |
| create | Determines whether cluster resources will be created | bool |
true |
no |
| create_cloudwatch_log_group | Determines whether to create a CloudWatch log group | bool |
true |
no |
| create_cluster_policy | Determines whether to create an MSK cluster policy | bool |
false |
no |
| create_configuration | Determines whether to create a configuration | bool |
true |
no |
| create_connect_worker_configuration | Determines whether to create connect worker configuration | bool |
false |
no |
| create_schema_registry | Determines whether to create a Glue schema registry for managing Avro schemas for the cluster | bool |
true |
no |
| create_scram_secret_association | Determines whether to create SASL/SCRAM secret association | bool |
false |
no |
| enable_storage_autoscaling | Determines whether autoscaling is enabled for storage | bool |
true |
no |
| encryption_at_rest_kms_key_arn | You may specify a KMS key short ID or ARN (it will always output an ARN) to use for encrypting your data at rest. If no key is specified, an AWS managed KMS ('aws/msk' managed service) key will be used for encrypting the data at rest | string |
null |
no |
| encryption_in_transit_client_broker | Encryption setting for data in transit between clients and brokers. Valid values: TLS, TLS_PLAINTEXT, and PLAINTEXT. Default value is TLS |
string |
null |
no |
| encryption_in_transit_in_cluster | Whether data communication among broker nodes is encrypted. Default value: true |
bool |
null |
no |
| enhanced_monitoring | Specify the desired enhanced MSK CloudWatch monitoring level. See Monitoring Amazon MSK with Amazon CloudWatch | string |
null |
no |
| firehose_delivery_stream | Name of the Kinesis Data Firehose delivery stream to deliver logs to | string |
null |
no |
| firehose_logs_enabled | Indicates whether you want to enable or disable streaming broker logs to Kinesis Data Firehose | bool |
false |
no |
| jmx_exporter_enabled | Indicates whether you want to enable or disable the JMX Exporter | bool |
false |
no |
| kafka_version | Specify the desired Kafka software version | string |
null |
no |
| name | Name of the MSK cluster | string |
"msk" |
no |
| node_exporter_enabled | Indicates whether you want to enable or disable the Node Exporter | bool |
false |
no |
| number_of_broker_nodes | The desired total number of broker nodes in the kafka cluster. It must be a multiple of the number of specified client subnets | number |
null |
no |
| s3_logs_bucket | Name of the S3 bucket to deliver logs to | string |
null |
no |
| s3_logs_enabled | Indicates whether you want to enable or disable streaming broker logs to S3 | bool |
false |
no |
| s3_logs_prefix | Prefix to append to the folder name | string |
null |
no |
| scaling_max_capacity | Max storage capacity for Kafka broker autoscaling | number |
250 |
no |
| scaling_role_arn | The ARN of the IAM role that allows Application AutoScaling to modify your scalable target on your behalf. This defaults to an IAM Service-Linked Role | string |
null |
no |
| scaling_target_value | The Kafka broker storage utilization at which scaling is initiated | number |
70 |
no |
| schema_registries | A map of schema registries to be created | map(any) |
{} |
no |
| schemas | A map schemas to be created within the schema registry | map(any) |
{} |
no |
| scram_secret_association_secret_arn_list | List of AWS Secrets Manager secret ARNs to associate with SCRAM | list(string) |
[] |
no |
| storage_mode | Controls storage mode for supported storage tiers. Valid values are: LOCAL or TIERED |
string |
null |
no |
| tags | A map of tags to assign to the resources created | map(string) |
{} |
no |
| timeouts | Create, update, and delete timeout configurations for the cluster | map(string) |
{} |
no |
| vpc_connections | Map of VPC Connections to create | any |
{} |
no |
| Name | Description |
|---|---|
| appautoscaling_policy_arn | The ARN assigned by AWS to the scaling policy |
| appautoscaling_policy_name | The scaling policy's name |
| appautoscaling_policy_policy_type | The scaling policy's type |
| arn | Amazon Resource Name (ARN) of the MSK cluster |
| bootstrap_brokers | Comma separated list of one or more hostname:port pairs of kafka brokers suitable to bootstrap connectivity to the kafka cluster |
| bootstrap_brokers_plaintext | Comma separated list of one or more hostname:port pairs of kafka brokers suitable to bootstrap connectivity to the kafka cluster. Contains a value if encryption_in_transit_client_broker is set to PLAINTEXT or TLS_PLAINTEXT |
| bootstrap_brokers_sasl_iam | One or more DNS names (or IP addresses) and SASL IAM port pairs. This attribute will have a value if encryption_in_transit_client_broker is set to TLS_PLAINTEXT or TLS and client_authentication_sasl_iam is set to true |
| bootstrap_brokers_sasl_scram | One or more DNS names (or IP addresses) and SASL SCRAM port pairs. This attribute will have a value if encryption_in_transit_client_broker is set to TLS_PLAINTEXT or TLS and client_authentication_sasl_scram is set to true |
| bootstrap_brokers_tls | One or more DNS names (or IP addresses) and TLS port pairs. This attribute will have a value if encryption_in_transit_client_broker is set to TLS_PLAINTEXT or TLS |
| cluster_uuid | UUID of the MSK cluster, for use in IAM policies |
| configuration_arn | Amazon Resource Name (ARN) of the configuration |
| configuration_latest_revision | Latest revision of the configuration |
| connect_custom_plugins | A map of output attributes for the connect custom plugins created |
| connect_worker_configuration_arn | The Amazon Resource Name (ARN) of the worker configuration |
| connect_worker_configuration_latest_revision | An ID of the latest successfully created revision of the worker configuration |
| current_version | Current version of the MSK Cluster used for updates, e.g. K13V1IB3VIYZZH |
| log_group_arn | The Amazon Resource Name (ARN) specifying the log group |
| schema_registries | A map of output attributes for the schema registries created |
| schemas | A map of output attributes for the schemas created |
| scram_secret_association_id | Amazon Resource Name (ARN) of the MSK cluster |
| vpc_connections | A map of output attributes for the VPC connections created |
| zookeeper_connect_string | A comma separated list of one or more hostname:port pairs to use to connect to the Apache Zookeeper cluster. The returned values are sorted alphabetically |
| zookeeper_connect_string_tls | A comma separated list of one or more hostname:port pairs to use to connect to the Apache Zookeeper cluster via TLS. The returned values are sorted alphabetically |
Apache-2.0 Licensed. See LICENSE.