From f2b4bfe461c59d25c2aac17d1bdbab6a71e8e7a7 Mon Sep 17 00:00:00 2001 From: "Byungjin Park (Claud)" Date: Thu, 2 Feb 2023 20:33:02 +0900 Subject: [PATCH 01/12] Use module instead of resource for resource group (#29) --- modules/msk-cluster/README.md | 6 ++-- modules/msk-cluster/migrations.tf | 5 ++++ modules/msk-cluster/resource-group.tf | 29 +++++-------------- modules/s3-archive-bucket/README.md | 9 +++--- modules/s3-archive-bucket/migrations.tf | 5 ++++ modules/s3-archive-bucket/resource-group.tf | 29 +++++-------------- modules/s3-archive-bucket/versions.tf | 2 +- modules/sqs-aws-event-queue/README.md | 9 +++--- modules/sqs-aws-event-queue/resource-group.tf | 29 +++++-------------- modules/sqs-aws-event-queue/versions.tf | 2 +- 10 files changed, 49 insertions(+), 76 deletions(-) create mode 100644 modules/msk-cluster/migrations.tf create mode 100644 modules/s3-archive-bucket/migrations.tf diff --git a/modules/msk-cluster/README.md b/modules/msk-cluster/README.md index 37deaa0..55e1239 100644 --- a/modules/msk-cluster/README.md +++ b/modules/msk-cluster/README.md @@ -22,13 +22,14 @@ This module creates following resources. | Name | Version | |------|---------| -| [aws](#provider\_aws) | 4.22.0 | -| [random](#provider\_random) | 3.3.2 | +| [aws](#provider\_aws) | 4.52.0 | +| [random](#provider\_random) | 3.4.3 | ## Modules | Name | Source | Version | |------|--------|---------| +| [resource\_group](#module\_resource\_group) | tedilabs/misc/aws//modules/resource-group | ~> 0.10.0 | | [secret](#module\_secret) | tedilabs/secret/aws//modules/secrets-manager-secret | ~> 0.2.0 | | [security\_group](#module\_security\_group) | tedilabs/network/aws//modules/security-group | 0.26.0 | @@ -39,7 +40,6 @@ This module creates following resources. | [aws_msk_cluster.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_cluster) | resource | | [aws_msk_configuration.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_configuration) | resource | | [aws_msk_scram_secret_association.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_scram_secret_association) | resource | -| [aws_resourcegroups_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/resourcegroups_group) | resource | | [random_password.this](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | | [aws_msk_broker_nodes.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/msk_broker_nodes) | data source | | [aws_subnet.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet) | data source | diff --git a/modules/msk-cluster/migrations.tf b/modules/msk-cluster/migrations.tf new file mode 100644 index 0000000..83a0f7e --- /dev/null +++ b/modules/msk-cluster/migrations.tf @@ -0,0 +1,5 @@ +# 2023-02-01 +moved { + from = aws_resourcegroups_group.this[0] + to = module.resource_group[0].aws_resourcegroups_group.this +} diff --git a/modules/msk-cluster/resource-group.tf b/modules/msk-cluster/resource-group.tf index af108f9..7487ba0 100644 --- a/modules/msk-cluster/resource-group.tf +++ b/modules/msk-cluster/resource-group.tf @@ -7,37 +7,24 @@ locals { replace(local.metadata.name, "/[^a-zA-Z0-9_\\.-]/", "-"), ]) ) - resource_group_filters = [ - for key, value in local.module_tags : { - "Key" = key - "Values" = [value] - } - ] - resource_group_query = <<-JSON - { - "ResourceTypeFilters": [ - "AWS::AllSupported" - ], - "TagFilters": ${jsonencode(local.resource_group_filters)} - } - JSON } -resource "aws_resourcegroups_group" "this" { + +module "resource_group" { + source = "tedilabs/misc/aws//modules/resource-group" + version = "~> 0.10.0" + count = (var.resource_group_enabled && var.module_tags_enabled) ? 1 : 0 name = local.resource_group_name description = var.resource_group_description - resource_query { - type = "TAG_FILTERS_1_0" - query = local.resource_group_query + query = { + resource_tags = local.module_tags } + module_tags_enabled = false tags = merge( - { - "Name" = local.resource_group_name - }, local.module_tags, var.tags, ) diff --git a/modules/s3-archive-bucket/README.md b/modules/s3-archive-bucket/README.md index cbbae13..53eacb9 100644 --- a/modules/s3-archive-bucket/README.md +++ b/modules/s3-archive-bucket/README.md @@ -18,24 +18,25 @@ This module creates following resources. | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.1 | +| [terraform](#requirement\_terraform) | >= 1.2 | | [aws](#requirement\_aws) | >= 4.16 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | 4.16.0 | +| [aws](#provider\_aws) | 4.52.0 | ## Modules -No modules. +| Name | Source | Version | +|------|--------|---------| +| [resource\_group](#module\_resource\_group) | tedilabs/misc/aws//modules/resource-group | ~> 0.10.0 | ## Resources | Name | Type | |------|------| -| [aws_resourcegroups_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/resourcegroups_group) | resource | | [aws_s3_bucket.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource | | [aws_s3_bucket_accelerate_configuration.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_accelerate_configuration) | resource | | [aws_s3_bucket_acl.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_acl) | resource | diff --git a/modules/s3-archive-bucket/migrations.tf b/modules/s3-archive-bucket/migrations.tf new file mode 100644 index 0000000..83a0f7e --- /dev/null +++ b/modules/s3-archive-bucket/migrations.tf @@ -0,0 +1,5 @@ +# 2023-02-01 +moved { + from = aws_resourcegroups_group.this[0] + to = module.resource_group[0].aws_resourcegroups_group.this +} diff --git a/modules/s3-archive-bucket/resource-group.tf b/modules/s3-archive-bucket/resource-group.tf index af108f9..7487ba0 100644 --- a/modules/s3-archive-bucket/resource-group.tf +++ b/modules/s3-archive-bucket/resource-group.tf @@ -7,37 +7,24 @@ locals { replace(local.metadata.name, "/[^a-zA-Z0-9_\\.-]/", "-"), ]) ) - resource_group_filters = [ - for key, value in local.module_tags : { - "Key" = key - "Values" = [value] - } - ] - resource_group_query = <<-JSON - { - "ResourceTypeFilters": [ - "AWS::AllSupported" - ], - "TagFilters": ${jsonencode(local.resource_group_filters)} - } - JSON } -resource "aws_resourcegroups_group" "this" { + +module "resource_group" { + source = "tedilabs/misc/aws//modules/resource-group" + version = "~> 0.10.0" + count = (var.resource_group_enabled && var.module_tags_enabled) ? 1 : 0 name = local.resource_group_name description = var.resource_group_description - resource_query { - type = "TAG_FILTERS_1_0" - query = local.resource_group_query + query = { + resource_tags = local.module_tags } + module_tags_enabled = false tags = merge( - { - "Name" = local.resource_group_name - }, local.module_tags, var.tags, ) diff --git a/modules/s3-archive-bucket/versions.tf b/modules/s3-archive-bucket/versions.tf index 6235d27..5242434 100644 --- a/modules/s3-archive-bucket/versions.tf +++ b/modules/s3-archive-bucket/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.1" + required_version = ">= 1.2" required_providers { aws = { diff --git a/modules/sqs-aws-event-queue/README.md b/modules/sqs-aws-event-queue/README.md index d5f7817..954be30 100644 --- a/modules/sqs-aws-event-queue/README.md +++ b/modules/sqs-aws-event-queue/README.md @@ -10,24 +10,25 @@ This module creates following resources. | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 0.15 | +| [terraform](#requirement\_terraform) | >= 1.2 | | [aws](#requirement\_aws) | >= 3.45 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | 3.69.0 | +| [aws](#provider\_aws) | 4.52.0 | ## Modules -No modules. +| Name | Source | Version | +|------|--------|---------| +| [resource\_group](#module\_resource\_group) | tedilabs/misc/aws//modules/resource-group | ~> 0.10.0 | ## Resources | Name | Type | |------|------| -| [aws_resourcegroups_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/resourcegroups_group) | resource | | [aws_sqs_queue.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource | | [aws_sqs_queue_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource | | [aws_iam_policy_document.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | diff --git a/modules/sqs-aws-event-queue/resource-group.tf b/modules/sqs-aws-event-queue/resource-group.tf index af108f9..7487ba0 100644 --- a/modules/sqs-aws-event-queue/resource-group.tf +++ b/modules/sqs-aws-event-queue/resource-group.tf @@ -7,37 +7,24 @@ locals { replace(local.metadata.name, "/[^a-zA-Z0-9_\\.-]/", "-"), ]) ) - resource_group_filters = [ - for key, value in local.module_tags : { - "Key" = key - "Values" = [value] - } - ] - resource_group_query = <<-JSON - { - "ResourceTypeFilters": [ - "AWS::AllSupported" - ], - "TagFilters": ${jsonencode(local.resource_group_filters)} - } - JSON } -resource "aws_resourcegroups_group" "this" { + +module "resource_group" { + source = "tedilabs/misc/aws//modules/resource-group" + version = "~> 0.10.0" + count = (var.resource_group_enabled && var.module_tags_enabled) ? 1 : 0 name = local.resource_group_name description = var.resource_group_description - resource_query { - type = "TAG_FILTERS_1_0" - query = local.resource_group_query + query = { + resource_tags = local.module_tags } + module_tags_enabled = false tags = merge( - { - "Name" = local.resource_group_name - }, local.module_tags, var.tags, ) diff --git a/modules/sqs-aws-event-queue/versions.tf b/modules/sqs-aws-event-queue/versions.tf index 6078ceb..dd959fc 100644 --- a/modules/sqs-aws-event-queue/versions.tf +++ b/modules/sqs-aws-event-queue/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 0.15" + required_version = ">= 1.2" required_providers { aws = { From dd9d222168bd7726f6ad4d52d85f19a9fafa9a1c Mon Sep 17 00:00:00 2001 From: Byungjin Park Date: Fri, 24 Feb 2023 02:15:03 +0900 Subject: [PATCH 02/12] Hello, 2023 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7ab0281..980fdc3 100644 --- a/README.md +++ b/README.md @@ -16,4 +16,4 @@ Like this project? Follow the repository on [GitHub](https://github.com/tedilabs Provided under the terms of the [Apache License](LICENSE). -Copyright © 2021-2022, [Byungjin Park](https://www.posquit0.com). +Copyright © 2021-2023, [Byungjin Park](https://www.posquit0.com). From 4c77ccf7605c1cac8255e5d90ad1938a9feebeb3 Mon Sep 17 00:00:00 2001 From: "Byungjin Park (Claud)" Date: Fri, 24 Feb 2023 02:56:17 +0900 Subject: [PATCH 03/12] Introduce tflint (#30) --- .github/workflows/integration.yaml | 101 ------------------ .github/workflows/terraform.integration.yaml | 104 +++++++++++++++++++ .github/workflows/yaml.integration.yaml | 58 +++++++++++ .pre-commit-config.yaml | 14 ++- .tflint.hcl | 77 ++++++++++++++ 5 files changed, 252 insertions(+), 102 deletions(-) delete mode 100644 .github/workflows/integration.yaml create mode 100644 .github/workflows/terraform.integration.yaml create mode 100644 .github/workflows/yaml.integration.yaml create mode 100644 .tflint.hcl diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml deleted file mode 100644 index e915d59..0000000 --- a/.github/workflows/integration.yaml +++ /dev/null @@ -1,101 +0,0 @@ -name: Integration - -on: - push: - branches: - - main - pull_request: {} - -concurrency: - group: integration-${{ github.ref }} - cancel-in-progress: true - -jobs: - changed: - runs-on: ubuntu-latest - - outputs: - terraform_modules_changed: ${{ steps.filter-terraform-modules.outputs.changed }} - terraform_modules_files: ${{ steps.filter-terraform-modules.outputs.files }} - terraform_modules_dirs: ${{ steps.filter-terraform-modules.outputs.dirs }} - yaml_changed: ${{ steps.filter-yaml.outputs.changed }} - yaml_files: ${{ steps.filter-yaml.outputs.files }} - - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Get Changed Files - id: changed-files - uses: dorny/paths-filter@v2 - with: - list-files: json - filters: | - modules: - - 'modules/**' - yaml: - - '**/*.yaml' - - '**/*.yml' - - - name: Filter changed Terraform Modules files to outputs - id: filter-terraform-modules - run: | - dirs=$(echo '${{ steps.changed-files.outputs.modules_files }}' | jq '[.[] | match("modules/[^/]+").string] | unique') - echo ::set-output name=changed::${{ steps.changed-files.outputs.modules }} - echo ::set-output name=files::${{ steps.changed-files.outputs.modules_files }} - echo ::set-output name=dirs::$dirs - - - name: Filter changed YAML files to outputs - id: filter-yaml - run: | - echo ::set-output name=changed::${{ steps.changed-files.outputs.yaml }} - echo ::set-output name=files::${{ steps.changed-files.outputs.yaml_files }} - - - terraform: - needs: - - changed - if: ${{ needs.changed.outputs.terraform_modules_changed != 'false' }} - runs-on: ubuntu-latest - - strategy: - matrix: - path: ${{ fromJson(needs.changed.outputs.terraform_modules_dirs) }} - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set-up terraform - uses: hashicorp/setup-terraform@v2 - - - name: Terraform fmt - id: terraform-fmt - working-directory: ${{ matrix.path }} - run: terraform fmt -check - continue-on-error: true - - - name: Terraform Validate - id: terraform-validate - working-directory: ${{ matrix.path }} - run: | - terraform init -backend=false - terraform validate -no-color - - - yaml: - needs: - - changed - if: ${{ needs.changed.outputs.yaml_changed != 'false' }} - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Lint YAML Files - id: yaml-lint - run: | - yamllint . diff --git a/.github/workflows/terraform.integration.yaml b/.github/workflows/terraform.integration.yaml new file mode 100644 index 0000000..b4cfa94 --- /dev/null +++ b/.github/workflows/terraform.integration.yaml @@ -0,0 +1,104 @@ +name: Integration (Terraform) + +on: + push: + branches: + - main + pull_request: {} + +concurrency: + group: terraform-integration-${{ github.ref }} + cancel-in-progress: true + +jobs: + changed: + name: Filter Changed Files and Directories + runs-on: ubuntu-latest + + outputs: + changed: ${{ steps.set-outputs.outputs.changed }} + modified: ${{ steps.set-outputs.outputs.modified }} + changed_files: ${{ steps.set-outputs.outputs.changed_files }} + modified_files: ${{ steps.set-outputs.outputs.modified_files }} + changed_directories: ${{ steps.set-outputs.outputs.changed_directories }} + modified_directories: ${{ steps.set-outputs.outputs.modified_directories }} + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Get Changed Files + id: changed-files + uses: tj-actions/changed-files@v35 + with: + files: | + modules/** + json: true + + - name: Get Changed Directories + id: changed-directories + uses: tj-actions/changed-files@v35 + with: + files: | + modules/** + examples/** + dir_names: "true" + dir_names_max_depth: 2 + json: true + + - name: Set outputs + id: set-outputs + run: | + echo "changed=${{ steps.changed-directories.outputs.any_changed }}" >> $GITHUB_OUTPUT + echo "modified=${{ steps.changed-directories.outputs.any_modified }}" >> $GITHUB_OUTPUT + + echo "changed_files=${{ steps.changed-files.outputs.all_changed_files }}" >> $GITHUB_OUTPUT + echo "modified_files=${{ steps.changed-files.outputs.all_modified_files }}" >> $GITHUB_OUTPUT + + echo "changed_directories=${{ steps.changed-directories.outputs.all_changed_files }}" >> $GITHUB_OUTPUT + echo "modified_directories=${{ steps.changed-directories.outputs.all_modified_files }}" >> $GITHUB_OUTPUT + + + terraform: + name: Lint (terraform) + needs: + - changed + if: ${{ needs.changed.outputs.modified == 'true' }} + uses: tedilabs/.github/.github/workflows/terraform.terraform.yaml@main + + strategy: + matrix: + path: ${{ fromJson(needs.changed.outputs.modified_directories) }} + + with: + terraform_target_dir: ${{ matrix.path }} + terraform_version: latest + terraform_host: app.terraform.io + secrets: + terraform_token: ${{ secrets.TERRAFORM_TOKEN }} + + + tflint: + name: Lint (tflint) + needs: + - changed + if: ${{ needs.changed.outputs.modified == 'true' }} + uses: tedilabs/.github/.github/workflows/terraform.tflint.yaml@main + + strategy: + matrix: + path: ${{ fromJson(needs.changed.outputs.modified_directories) }} + + with: + tflint_version: latest + tflint_config_file: .tflint.hcl + tflint_target_dir: ${{ matrix.path }} + tflint_recursive_enabled: false + tflint_terraform_init_enabled: true + terraform_version: latest + terraform_host: app.terraform.io + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + terraform_token: ${{ secrets.TERRAFORM_TOKEN }} diff --git a/.github/workflows/yaml.integration.yaml b/.github/workflows/yaml.integration.yaml new file mode 100644 index 0000000..20342a1 --- /dev/null +++ b/.github/workflows/yaml.integration.yaml @@ -0,0 +1,58 @@ +name: Integration (YAML) + +on: + push: + branches: + - main + pull_request: {} + +concurrency: + group: yaml-integration-${{ github.ref }} + cancel-in-progress: true + +jobs: + changed: + name: Filter Changed Files and Directories + runs-on: ubuntu-latest + + outputs: + changed: ${{ steps.set-outputs.outputs.changed }} + modified: ${{ steps.set-outputs.outputs.modified }} + changed_files: ${{ steps.set-outputs.outputs.changed_files }} + modified_files: ${{ steps.set-outputs.outputs.modified_files }} + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Get Changed Files + id: changed-files + uses: tj-actions/changed-files@v35 + with: + files: | + **/*.yaml + **/*.yml + json: true + + - name: Set outputs + id: set-outputs + run: | + echo "changed=${{ steps.changed-files.outputs.any_changed }}" >> $GITHUB_OUTPUT + echo "modified=${{ steps.changed-files.outputs.any_modified }}" >> $GITHUB_OUTPUT + + echo "changed_files=${{ steps.changed-files.outputs.all_changed_files }}" >> $GITHUB_OUTPUT + echo "modified_files=${{ steps.changed-files.outputs.all_modified_files }}" >> $GITHUB_OUTPUT + + lint: + name: Lint (yamllint) + needs: + - changed + if: ${{ needs.changed.outputs.modified == 'true' }} + uses: tedilabs/.github/.github/workflows/yaml.yamllint.yaml@main + + with: + yamllint_version: latest + yamllint_config_file: .yamllint.yaml + yamllint_target_dir: ./ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aa33d01..d0b1fea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,8 +1,20 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.70.1 + rev: v1.77.1 hooks: - id: terraform_fmt + args: + - --args=-diff - id: terraform_validate + args: + - --hook-config=--retry-once-with-cleanup=true + - id: terraform_tflint + args: + - --args=--config=__GIT_WORKING_DIR__/.tflint.hcl + files: ^modules/ - id: terraform_docs args: ["--args=--sort-by required"] +- repo: https://github.com/adrienverge/yamllint + rev: v1.29.0 + hooks: + - id: yamllint diff --git a/.tflint.hcl b/.tflint.hcl new file mode 100644 index 0000000..0788789 --- /dev/null +++ b/.tflint.hcl @@ -0,0 +1,77 @@ +config { + plugin_dir = "~/.tflint.d/plugins" + + format = "compact" + module = true + force = false + disabled_by_default = false + + ignore_module = {} +} + + +################################################### +# Rule Sets - Terraform +################################################### + +plugin "terraform" { + enabled = true + preset = "recommended" +} + +rule "terraform_comment_syntax" { + enabled = true +} + +rule "terraform_documented_variables" { + enabled = true +} + +rule "terraform_documented_outputs" { + enabled = true +} + +rule "terraform_naming_convention" { + enabled = true + format = "snake_case" + + custom_formats = { + extended_snake_case = { + description = "Extended snake_case Format which allows double underscore like `a__b`." + regex = "^[a-z][a-z0-9]+([_]{1,2}[a-z0-9]+)*$" + } + } + + module { + format = "extended_snake_case" + } + + resource { + format = "extended_snake_case" + } + + data { + format = "extended_snake_case" + } +} + +rule "terraform_unused_declarations" { + enabled = false +} + +rule "terraform_unused_required_providers" { + enabled = true +} + + +################################################### +# Rule Sets - AWS +################################################### + +plugin "aws" { + source = "github.com/terraform-linters/tflint-ruleset-aws" + version = "0.21.1" + + enabled = true + deep_check = false +} From 308272807e71be9b5a26dd76dba1bc54f9543ecc Mon Sep 17 00:00:00 2001 From: "Byungjin Park (Claud)" Date: Fri, 24 Feb 2023 03:05:24 +0900 Subject: [PATCH 04/12] Fix tflint failed code (#31) --- modules/msk-cluster/cluster.tf | 2 +- modules/msk-cluster/outputs.tf | 48 +++++++++++++++++----------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/modules/msk-cluster/cluster.tf b/modules/msk-cluster/cluster.tf index c174a1b..0d83526 100644 --- a/modules/msk-cluster/cluster.tf +++ b/modules/msk-cluster/cluster.tf @@ -55,7 +55,7 @@ resource "aws_msk_cluster" "this" { az_distribution = "DEFAULT" client_subnets = var.broker_subnets security_groups = concat( - module.security_group.*.id, + module.security_group[*].id, var.broker_additional_security_groups ) diff --git a/modules/msk-cluster/outputs.tf b/modules/msk-cluster/outputs.tf index fd44387..844179e 100644 --- a/modules/msk-cluster/outputs.tf +++ b/modules/msk-cluster/outputs.tf @@ -30,7 +30,7 @@ output "kafka_config" { output "broker_security_group_id" { description = "The id of security group that were created for the MSK cluster." - value = try(module.security_group.*.id[0], null) + value = try(module.security_group[*].id[0], null) } output "broker_nodes" { @@ -51,18 +51,18 @@ output "broker" { EOF value = { size = aws_msk_cluster.this.number_of_broker_nodes - instance_type = aws_msk_cluster.this.broker_node_group_info.0.instance_type + instance_type = aws_msk_cluster.this.broker_node_group_info[0].instance_type - subnets = aws_msk_cluster.this.broker_node_group_info.0.client_subnets + subnets = aws_msk_cluster.this.broker_node_group_info[0].client_subnets public_access_enabled = var.broker_public_access_enabled - security_groups = aws_msk_cluster.this.broker_node_group_info.0.security_groups - default_security_group_id = try(module.security_group.*.id[0], null) + security_groups = aws_msk_cluster.this.broker_node_group_info[0].security_groups + default_security_group_id = try(module.security_group[*].id[0], null) volume = { - size = aws_msk_cluster.this.broker_node_group_info.0.storage_info.0.ebs_storage_info.0.volume_size + size = aws_msk_cluster.this.broker_node_group_info[0].storage_info[0].ebs_storage_info[0].volume_size provisioned_throughput = { - enabled = try(aws_msk_cluster.this.broker_node_group_info.0.storage_info.0.ebs_storage_info.0.provisioned_throughput.0.enabled, false) - throughput = try(aws_msk_cluster.this.broker_node_group_info.0.storage_info.0.ebs_storage_info.0.provisioned_throughput.0.volume_throughput, null) + enabled = try(aws_msk_cluster.this.broker_node_group_info[0].storage_info[0].ebs_storage_info[0].provisioned_throughput[0].enabled, false) + throughput = try(aws_msk_cluster.this.broker_node_group_info[0].storage_info[0].ebs_storage_info[0].provisioned_throughput[0].volume_throughput, null) } } } @@ -72,21 +72,21 @@ output "auth" { description = "A configuration for authentication of the Kafka cluster." value = { unauthenticated_access = { - enabled = aws_msk_cluster.this.client_authentication.0.unauthenticated + enabled = aws_msk_cluster.this.client_authentication[0].unauthenticated } sasl = { iam = { - enabled = aws_msk_cluster.this.client_authentication.0.sasl.0.iam + enabled = aws_msk_cluster.this.client_authentication[0].sasl[0].iam } scram = { - enabled = aws_msk_cluster.this.client_authentication.0.sasl.0.scram + enabled = aws_msk_cluster.this.client_authentication[0].sasl[0].scram kms_key = var.auth_sasl_scram_kms_key users = var.auth_sasl_scram_users } } tls = { enabled = var.auth_tls_enabled - acm_ca_arns = try(aws_msk_cluster.this.client_authentication.0.tls.0.certificate_authority_arns, []) + acm_ca_arns = try(aws_msk_cluster.this.client_authentication[0].tls[0].certificate_authority_arns, []) } } } @@ -99,11 +99,11 @@ output "encryption" { EOF value = { at_rest = { - kms_key = aws_msk_cluster.this.encryption_info.0.encryption_at_rest_kms_key_arn + kms_key = aws_msk_cluster.this.encryption_info[0].encryption_at_rest_kms_key_arn } in_transit = { - in_cluster_enabled = aws_msk_cluster.this.encryption_info.0.encryption_in_transit.0.in_cluster - client_mode = aws_msk_cluster.this.encryption_info.0.encryption_in_transit.0.client_broker + in_cluster_enabled = aws_msk_cluster.this.encryption_info[0].encryption_in_transit[0].in_cluster + client_mode = aws_msk_cluster.this.encryption_info[0].encryption_in_transit[0].client_broker } } } @@ -117,17 +117,17 @@ output "logging" { EOF value = { cloudwatch = { - enabled = aws_msk_cluster.this.logging_info.0.broker_logs.0.cloudwatch_logs.0.enabled - log_group = aws_msk_cluster.this.logging_info.0.broker_logs.0.cloudwatch_logs.0.log_group + enabled = aws_msk_cluster.this.logging_info[0].broker_logs[0].cloudwatch_logs[0].enabled + log_group = aws_msk_cluster.this.logging_info[0].broker_logs[0].cloudwatch_logs[0].log_group } firehose = { - enabled = aws_msk_cluster.this.logging_info.0.broker_logs.0.firehose.0.enabled - delivery_stream = aws_msk_cluster.this.logging_info.0.broker_logs.0.firehose.0.delivery_stream + enabled = aws_msk_cluster.this.logging_info[0].broker_logs[0].firehose[0].enabled + delivery_stream = aws_msk_cluster.this.logging_info[0].broker_logs[0].firehose[0].delivery_stream } s3 = { - enabled = aws_msk_cluster.this.logging_info.0.broker_logs.0.s3.0.enabled - bucket = aws_msk_cluster.this.logging_info.0.broker_logs.0.s3.0.bucket - prefix = aws_msk_cluster.this.logging_info.0.broker_logs.0.s3.0.prefix + enabled = aws_msk_cluster.this.logging_info[0].broker_logs[0].s3[0].enabled + bucket = aws_msk_cluster.this.logging_info[0].broker_logs[0].s3[0].bucket + prefix = aws_msk_cluster.this.logging_info[0].broker_logs[0].s3[0].prefix } } } @@ -143,8 +143,8 @@ output "monitoring" { level = aws_msk_cluster.this.enhanced_monitoring } prometheus = { - jmx_exporter_enabled = aws_msk_cluster.this.open_monitoring.0.prometheus.0.jmx_exporter.0.enabled_in_broker - node_exporter_enabled = aws_msk_cluster.this.open_monitoring.0.prometheus.0.node_exporter.0.enabled_in_broker + jmx_exporter_enabled = aws_msk_cluster.this.open_monitoring[0].prometheus[0].jmx_exporter[0].enabled_in_broker + node_exporter_enabled = aws_msk_cluster.this.open_monitoring[0].prometheus[0].node_exporter[0].enabled_in_broker } } } From c25ca6414ec26016442b3c9373a3c4ef3da72143 Mon Sep 17 00:00:00 2001 From: Byungjin Park Date: Sun, 26 Feb 2023 22:45:44 +0900 Subject: [PATCH 05/12] Update workflow --- .github/workflows/terraform.integration.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/terraform.integration.yaml b/.github/workflows/terraform.integration.yaml index b4cfa94..2fc3c46 100644 --- a/.github/workflows/terraform.integration.yaml +++ b/.github/workflows/terraform.integration.yaml @@ -35,6 +35,7 @@ jobs: with: files: | modules/** + examples/** json: true - name: Get Changed Directories From af059ce2011383570d6420970e701438ae5e3b37 Mon Sep 17 00:00:00 2001 From: Byungjin Park Date: Mon, 6 Mar 2023 02:39:18 +0900 Subject: [PATCH 06/12] Update github workflows --- .github/workflows/terraform.integration.yaml | 3 +++ .github/workflows/yaml.integration.yaml | 2 ++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/terraform.integration.yaml b/.github/workflows/terraform.integration.yaml index 2fc3c46..a1ad44f 100644 --- a/.github/workflows/terraform.integration.yaml +++ b/.github/workflows/terraform.integration.yaml @@ -78,6 +78,8 @@ jobs: terraform_version: latest terraform_host: app.terraform.io secrets: + gh_token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.GITHUB_TOKEN }} terraform_token: ${{ secrets.TERRAFORM_TOKEN }} @@ -101,5 +103,6 @@ jobs: terraform_version: latest terraform_host: app.terraform.io secrets: + gh_token: ${{ secrets.GITHUB_TOKEN }} token: ${{ secrets.GITHUB_TOKEN }} terraform_token: ${{ secrets.TERRAFORM_TOKEN }} diff --git a/.github/workflows/yaml.integration.yaml b/.github/workflows/yaml.integration.yaml index 20342a1..751ecf9 100644 --- a/.github/workflows/yaml.integration.yaml +++ b/.github/workflows/yaml.integration.yaml @@ -56,3 +56,5 @@ jobs: yamllint_version: latest yamllint_config_file: .yamllint.yaml yamllint_target_dir: ./ + secrets: + token: ${{ secrets.GITHUB_TOKEN }} From 00516d034aa98453086deb894d82b188b8e7505f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 25 Jun 2023 23:59:02 +0900 Subject: [PATCH 07/12] Bump tj-actions/changed-files from 35 to 37 (#33) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/terraform.integration.yaml | 4 ++-- .github/workflows/yaml.integration.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/terraform.integration.yaml b/.github/workflows/terraform.integration.yaml index a1ad44f..1572c19 100644 --- a/.github/workflows/terraform.integration.yaml +++ b/.github/workflows/terraform.integration.yaml @@ -31,7 +31,7 @@ jobs: - name: Get Changed Files id: changed-files - uses: tj-actions/changed-files@v35 + uses: tj-actions/changed-files@v37 with: files: | modules/** @@ -40,7 +40,7 @@ jobs: - name: Get Changed Directories id: changed-directories - uses: tj-actions/changed-files@v35 + uses: tj-actions/changed-files@v37 with: files: | modules/** diff --git a/.github/workflows/yaml.integration.yaml b/.github/workflows/yaml.integration.yaml index 751ecf9..a81b49e 100644 --- a/.github/workflows/yaml.integration.yaml +++ b/.github/workflows/yaml.integration.yaml @@ -29,7 +29,7 @@ jobs: - name: Get Changed Files id: changed-files - uses: tj-actions/changed-files@v35 + uses: tj-actions/changed-files@v37 with: files: | **/*.yaml From 905fd3bbac10239c89bee6328c53bc808f18821c Mon Sep 17 00:00:00 2001 From: Byungjin Park Date: Tue, 27 Jun 2023 23:19:01 +0900 Subject: [PATCH 08/12] Update pre-commit config --- .pre-commit-config.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d0b1fea..10bde48 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,13 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.77.1 + rev: v1.81.0 hooks: - id: terraform_fmt args: - --args=-diff - id: terraform_validate args: + - --tf-init-args=-upgrade - --hook-config=--retry-once-with-cleanup=true - id: terraform_tflint args: @@ -14,7 +15,8 @@ repos: files: ^modules/ - id: terraform_docs args: ["--args=--sort-by required"] + - repo: https://github.com/adrienverge/yamllint - rev: v1.29.0 + rev: v1.32.0 hooks: - id: yamllint From 514e9de5113440793b8b0fa957a66a9f615061d5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 26 Aug 2023 19:23:22 +0900 Subject: [PATCH 09/12] Bump tj-actions/changed-files from 37 to 38 (#34) Bumps [tj-actions/changed-files](https://github.com/tj-actions/changed-files) from 37 to 38. - [Release notes](https://github.com/tj-actions/changed-files/releases) - [Changelog](https://github.com/tj-actions/changed-files/blob/main/HISTORY.md) - [Commits](https://github.com/tj-actions/changed-files/compare/v37...v38) --- updated-dependencies: - dependency-name: tj-actions/changed-files dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Byungjin Park (Claud) --- .github/workflows/terraform.integration.yaml | 4 ++-- .github/workflows/yaml.integration.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/terraform.integration.yaml b/.github/workflows/terraform.integration.yaml index 1572c19..8cfe55e 100644 --- a/.github/workflows/terraform.integration.yaml +++ b/.github/workflows/terraform.integration.yaml @@ -31,7 +31,7 @@ jobs: - name: Get Changed Files id: changed-files - uses: tj-actions/changed-files@v37 + uses: tj-actions/changed-files@v38 with: files: | modules/** @@ -40,7 +40,7 @@ jobs: - name: Get Changed Directories id: changed-directories - uses: tj-actions/changed-files@v37 + uses: tj-actions/changed-files@v38 with: files: | modules/** diff --git a/.github/workflows/yaml.integration.yaml b/.github/workflows/yaml.integration.yaml index a81b49e..5149a2d 100644 --- a/.github/workflows/yaml.integration.yaml +++ b/.github/workflows/yaml.integration.yaml @@ -29,7 +29,7 @@ jobs: - name: Get Changed Files id: changed-files - uses: tj-actions/changed-files@v37 + uses: tj-actions/changed-files@v38 with: files: | **/*.yaml From d16060339e8831267ff7cb5a9d2550419eb609f3 Mon Sep 17 00:00:00 2001 From: Byungjin Park Date: Sat, 2 Sep 2023 23:26:58 +0900 Subject: [PATCH 10/12] Migrate msk-cluster to terraform-aws-messaging --- .github/labeler.yaml | 3 - .github/labels.yaml | 3 - modules/msk-cluster/README.md | 108 ---------- modules/msk-cluster/cluster.tf | 172 ---------------- modules/msk-cluster/migrations.tf | 5 - modules/msk-cluster/outputs.tf | 185 ----------------- modules/msk-cluster/resource-group.tf | 31 --- modules/msk-cluster/scram-secrets.tf | 55 ----- modules/msk-cluster/security-group.tf | 124 ------------ modules/msk-cluster/variables.tf | 278 -------------------------- modules/msk-cluster/versions.tf | 14 -- 11 files changed, 978 deletions(-) delete mode 100644 modules/msk-cluster/README.md delete mode 100644 modules/msk-cluster/cluster.tf delete mode 100644 modules/msk-cluster/migrations.tf delete mode 100644 modules/msk-cluster/outputs.tf delete mode 100644 modules/msk-cluster/resource-group.tf delete mode 100644 modules/msk-cluster/scram-secrets.tf delete mode 100644 modules/msk-cluster/security-group.tf delete mode 100644 modules/msk-cluster/variables.tf delete mode 100644 modules/msk-cluster/versions.tf diff --git a/.github/labeler.yaml b/.github/labeler.yaml index 30da008..3977b4a 100644 --- a/.github/labeler.yaml +++ b/.github/labeler.yaml @@ -1,7 +1,4 @@ # Modules -":floppy_disk: msk-cluster": -- modules/msk-cluster/**/* - ":floppy_disk: resource-group": - modules/resource-group/**/* diff --git a/.github/labels.yaml b/.github/labels.yaml index b2de133..1272df1 100644 --- a/.github/labels.yaml +++ b/.github/labels.yaml @@ -40,9 +40,6 @@ name: "size/XL" # Modules -- color: "fbca04" - description: "This issue or pull request is related to msk-cluster module." - name: ":floppy_disk: msk-cluster" - color: "fbca04" description: "This issue or pull request is related to resource-group module." name: ":floppy_disk: resource-group" diff --git a/modules/msk-cluster/README.md b/modules/msk-cluster/README.md deleted file mode 100644 index 55e1239..0000000 --- a/modules/msk-cluster/README.md +++ /dev/null @@ -1,108 +0,0 @@ -# msk-cluster - -This module creates following resources. - -- `aws_msk_cluster` -- `aws_msk_configuration` -- `aws_msk_scram_secret_association` (optional) -- `aws_security_group` (optional) -- `aws_security_group_rule` (optional) -- `aws_secretsmanager_secret` (optional) - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.2 | -| [aws](#requirement\_aws) | >= 4.22 | -| [random](#requirement\_random) | >= 3.3 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | 4.52.0 | -| [random](#provider\_random) | 3.4.3 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [resource\_group](#module\_resource\_group) | tedilabs/misc/aws//modules/resource-group | ~> 0.10.0 | -| [secret](#module\_secret) | tedilabs/secret/aws//modules/secrets-manager-secret | ~> 0.2.0 | -| [security\_group](#module\_security\_group) | tedilabs/network/aws//modules/security-group | 0.26.0 | - -## Resources - -| Name | Type | -|------|------| -| [aws_msk_cluster.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_cluster) | resource | -| [aws_msk_configuration.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_configuration) | resource | -| [aws_msk_scram_secret_association.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_scram_secret_association) | resource | -| [random_password.this](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | -| [aws_msk_broker_nodes.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/msk_broker_nodes) | data source | -| [aws_subnet.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [broker\_size](#input\_broker\_size) | (Required) The desired total number of broker nodes in the kafka cluster. It must be a multiple of the number of specified client subnets. | `number` | n/a | yes | -| [broker\_subnets](#input\_broker\_subnets) | (Required) A list of subnet IDs to place ENIs of the MSK cluster broker nodes within. | `list(string)` | n/a | yes | -| [name](#input\_name) | (Required) Name of the MSK cluster. | `string` | n/a | yes | -| [auth\_sasl\_iam\_enabled](#input\_auth\_sasl\_iam\_enabled) | (Optional) Enables IAM client authentication. | `bool` | `false` | no | -| [auth\_sasl\_scram\_enabled](#input\_auth\_sasl\_scram\_enabled) | (Optional) Enables SCRAM client authentication via AWS Secrets Manager. | `bool` | `false` | no | -| [auth\_sasl\_scram\_kms\_key](#input\_auth\_sasl\_scram\_kms\_key) | (Optional) The ARN of a KMS key to encrypt AWS SeecretsManager Secret resources for storing SASL/SCRAM authentication data. Only required when the MSK cluster has SASL/SCRAM authentication enabled. The Username/Password Authentication based on SASL/SCRAM needs to create a Secret resource in AWS SecretsManager with a custom AWS KMS Key. A secret created with the default AWS KMS key cannot be used with an Amazon MSK cluster. | `string` | `null` | no | -| [auth\_sasl\_scram\_users](#input\_auth\_sasl\_scram\_users) | (Optional) A list of usernames to be allowed for SASL/SCRAM authentication to the MSK cluster. The password for each username is randomly generated and stored in AWS SecretsManager secret. | `set(string)` | `[]` | no | -| [auth\_tls\_acm\_ca\_arns](#input\_auth\_tls\_acm\_ca\_arns) | (Optional) List of ACM Certificate Authority Amazon Resource Names (ARNs). | `list(string)` | `[]` | no | -| [auth\_tls\_enabled](#input\_auth\_tls\_enabled) | (Optional) Enables TLS client authentication. | `bool` | `false` | no | -| [auth\_unauthenticated\_access\_enabled](#input\_auth\_unauthenticated\_access\_enabled) | (Optional) Enables unauthenticated access. Defaults to `true`. | `bool` | `true` | no | -| [broker\_additional\_security\_groups](#input\_broker\_additional\_security\_groups) | (Optional) A list of security group IDs to associate with ENIs to control who can communicate with the cluster. | `list(string)` | `[]` | no | -| [broker\_allowed\_ingress\_cidrs](#input\_broker\_allowed\_ingress\_cidrs) | (Optional) A list of CIDR for MSK ingress access. | `list(string)` | `[]` | no | -| [broker\_instance\_type](#input\_broker\_instance\_type) | (Optional) The instance type to use for the kafka brokers. | `string` | `"kafka.m5.large"` | no | -| [broker\_public\_access\_enabled](#input\_broker\_public\_access\_enabled) | (Optional) Whether to allow public access to MSK brokers. | `bool` | `false` | no | -| [broker\_volume\_provisioned\_throughput](#input\_broker\_volume\_provisioned\_throughput) | (Optional) Throughput value of the EBS volumes for the data drive on each kafka broker node in MiB per second. The minimum value is `250`. The maximum value varies between broker type. | `number` | `null` | no | -| [broker\_volume\_provisioned\_throughput\_enabled](#input\_broker\_volume\_provisioned\_throughput\_enabled) | (Optional) Whether provisioned throughput is enabled or not. You can specify the provisioned throughput rate in MiB per second for clusters whose brokers are of type `kafka.m5.4xlarge` or larger and if the storage volume is 10 GiB or greater. Defaults to `false`. | `bool` | `false` | no | -| [broker\_volume\_size](#input\_broker\_volume\_size) | (Optional) The size in GiB of the EBS volume for the data drive on each broker node. Minimum value of `1` and maximum value of `16384`. Defaults to `1000`. | `number` | `1000` | no | -| [encryption\_at\_rest\_kms\_key](#input\_encryption\_at\_rest\_kms\_key) | (Optional) Specify a KMS key short ID or ARN (it will always output an ARN) to use for encrypting your data at rest. If no key is specified, an AWS managed KMS ('aws/msk' managed service) key will be used for encrypting the data at rest. | `string` | `""` | no | -| [encryption\_in\_transit\_client\_mode](#input\_encryption\_in\_transit\_client\_mode) | (Optional) Encryption setting for data in transit between clients and brokers. `TLS`, `TLS_PLAINTEXT`, `PLAINTEXT` are available. | `string` | `"TLS_PLAINTEXT"` | no | -| [encryption\_in\_transit\_in\_cluster\_enabled](#input\_encryption\_in\_transit\_in\_cluster\_enabled) | (Optional) Whether data communication among broker nodes is encrypted. | `bool` | `true` | no | -| [kafka\_server\_properties](#input\_kafka\_server\_properties) | (Optional) Contents of the `server.properties` file for configuration of Kafka. | `map(string)` | `{}` | no | -| [kafka\_version](#input\_kafka\_version) | (Optional) Kafka version to use for the MSK cluster. | `string` | `"2.8.0"` | no | -| [logging\_cloudwatch\_enabled](#input\_logging\_cloudwatch\_enabled) | (Optional) Indicates whether you want to enable or disable streaming broker logs to Cloudwatch Logs. | `bool` | `false` | no | -| [logging\_cloudwatch\_log\_group](#input\_logging\_cloudwatch\_log\_group) | (Optional) The name of log group on CloudWatch Logs to deliver logs to. | `string` | `""` | no | -| [logging\_firehose\_delivery\_stream](#input\_logging\_firehose\_delivery\_stream) | (Optional) Name of the Kinesis Data Firehose delivery stream to deliver logs to. | `string` | `""` | no | -| [logging\_firehose\_enabled](#input\_logging\_firehose\_enabled) | (Optional) Indicates whether you want to enable or disable streaming broker logs to Kinesis Data Firehose. | `bool` | `false` | no | -| [logging\_s3\_bucket](#input\_logging\_s3\_bucket) | (Optional) The name of the S3 bucket to deliver logs to. | `string` | `""` | no | -| [logging\_s3\_enabled](#input\_logging\_s3\_enabled) | (Optional) Indicates whether you want to enable or disable streaming broker logs to S3. | `bool` | `false` | no | -| [logging\_s3\_prefix](#input\_logging\_s3\_prefix) | (Optional) The prefix to append to the folder name. | `string` | `""` | no | -| [module\_tags\_enabled](#input\_module\_tags\_enabled) | (Optional) Whether to create AWS Resource Tags for the module informations. | `bool` | `true` | no | -| [monitoring\_cloudwatch\_level](#input\_monitoring\_cloudwatch\_level) | (Optional) The desired enhanced MSK CloudWatch monitoring level. `DEFAULT`, `PER_BROKER`, `PER_TOPIC_PER_BROKER`, `PER_TOPIC_PER_PARTITION` are available. | `string` | `"DEFAULT"` | no | -| [monitoring\_prometheus\_jmx\_exporter\_enabled](#input\_monitoring\_prometheus\_jmx\_exporter\_enabled) | (Optional) Indicates whether you want to enable or disable the JMX Exporter. | `bool` | `false` | no | -| [monitoring\_prometheus\_node\_exporter\_enabled](#input\_monitoring\_prometheus\_node\_exporter\_enabled) | (Optional) Indicates whether you want to enable or disable the Node Exporter. | `bool` | `false` | no | -| [resource\_group\_description](#input\_resource\_group\_description) | (Optional) The description of Resource Group. | `string` | `"Managed by Terraform."` | no | -| [resource\_group\_enabled](#input\_resource\_group\_enabled) | (Optional) Whether to create Resource Group to find and group AWS resources which are created by this module. | `bool` | `true` | no | -| [resource\_group\_name](#input\_resource\_group\_name) | (Optional) The name of Resource Group. A Resource Group name can have a maximum of 127 characters, including letters, numbers, hyphens, dots, and underscores. The name cannot start with `AWS` or `aws`. | `string` | `""` | no | -| [tags](#input\_tags) | (Optional) A map of tags to add to all resources. | `map(string)` | `{}` | no | -| [timeouts](#input\_timeouts) | (Optional) How long to wait for the MSK cluster to be created/updated/deleted. | `map(string)` | 
{
  "create": "120m",
  "delete": "120m",
  "update": "120m"
}
| no | - -## Outputs - -| Name | Description | -|------|-------------| -| [arn](#output\_arn) | The ARN of the MSK cluster. | -| [auth](#output\_auth) | A configuration for authentication of the Kafka cluster. | -| [bootstrap\_brokers](#output\_bootstrap\_brokers) | A configuration for connecting to the Kafka cluster.
`plaintext` - A comma separated list of one or more hostname:port pairs of kafka brokers suitable to boostrap connectivity to the kafka cluster. Only contains value if `client_encryption_in_transit_mode` is set to PLAINTEXT or TLS\_PLAINTEXT. AWS may not always return all endpoints so the values may not be stable across applies.
`sasl_iam` - A comma separated list of one or more DNS names (or IPs) and SASL IAM port pairs. Only contains value if `client_encryption_in_transit_mode` is set to TLS\_PLAINTEXT or TLS. AWS may not always return all endpoints so the values may not be stable across applies.
`sasl_scram` - A comma separated list of one or more DNS names (or IPs) and SASL SCRAM port pairs. Only contains value if `client_encryption_in_transit_mode` is set to TLS\_PLAINTEXT or TLS. AWS may not always return all endpoints so the values may not be stable across applies.
`tls` - A comma separated list of one or more DNS names (or IPs) and TLS port pairs kafka brokers suitable to boostrap connectivity to the kafka cluster. Only contains value if `client_encryption_in_transit_mode is set to TLS_PLAINTEXT or TLS. AWS may not always return all endpoints so the values may not be stable across applies.
`public\_sasl\_iam` - A comma separated list of one or more DNS names (or IPs) and SASL IAM port pairs. Only contains value if `client\_encryption\_in\_transit\_mode` is set to TLS_PLAINTEXT or TLS and `auth\_sasl\_iam\_enabled` is `true` and `broker\_public\_access\_enabled` is `true`. AWS may not always return all endpoints so the values may not be stable across applies.
`public\_sasl\_scram` - A comma separated list of one or more DNS names (or IPs) and SASL SCRAM port pairs. Only contains value if `client\_encryption\_in\_transit\_mode` is set to TLS_PLAINTEXT or TLS and `auth\_sasl\_scram\_enabled` is `true` and `broker\_public\_access\_enabled` is `true`. AWS may not always return all endpoints so the values may not be stable across applies.
`public\_tls` - A comma separated list of one or more DNS names (or IPs) and TLS port pairs. Only contains value if `client\_encryption\_in\_transit\_mode` is set to TLS_PLAINTEXT or TLS and `broker\_public\_access\_enabled` is `true`. AWS may not always return all endpoints so the values may not be stable across applies.
` | -| [broker](#output\_broker) | A configuration for brokers of the Kafka cluster.
`size` - The number of broker nodes in the kafka cluster.
`instance_type` - The instance type used by the kafka brokers.

`public_access_enabled` - Whether public access to MSK brokers is enabled.
`security_groups` - A list of the security groups associated with the MSK cluster.

`volume` - A EBS volume information for MSK brokers. | -| [broker\_nodes](#output\_broker\_nodes) | The information of broker nodes in the kafka cluster. | -| [broker\_security\_group\_id](#output\_broker\_security\_group\_id) | The id of security group that were created for the MSK cluster. | -| [encryption](#output\_encryption) | A configuration for encryption of the Kafka cluster.
`at_rest` - The configuration for encryption at rest.
`in_transit` - The configuration for encryption in transit. | -| [kafka\_config](#output\_kafka\_config) | The MSK configuration. | -| [kafka\_version](#output\_kafka\_version) | The MSK cluster version. | -| [logging](#output\_logging) | A configuration for logging of the Kafka cluster.
`cloudwatch` - The configuration for MSK broker logs to CloudWatch Logs.
`firehose` - The configuration for MSK broker logs to Kinesis Firehose.
`s3` - The configuration for MSK broker logs to S3 Bucket. | -| [monitoring](#output\_monitoring) | A configuration for monitoring of the Kafka cluster.
`cloudwatch` - The configuration for MSK CloudWatch Metrics.
`prometheus` - The configuration for Prometheus open monitoring. | -| [name](#output\_name) | The MSK cluster name. | -| [version](#output\_version) | Current version of the MSK Cluster used for updates. | -| [zookeeper\_connections](#output\_zookeeper\_connections) | A configuration for connecting to the Apache Zookeeper cluster.
`tcp` - A comma separated list of one or more IP:port pairs to use to connect to the Apache Zookeeper cluster.
`tls` - A comma separated list of one or more IP:port pairs to use to connect to the Apache Zookeeper cluster via TLS. | - diff --git a/modules/msk-cluster/cluster.tf b/modules/msk-cluster/cluster.tf deleted file mode 100644 index 0d83526..0000000 --- a/modules/msk-cluster/cluster.tf +++ /dev/null @@ -1,172 +0,0 @@ -locals { - metadata = { - package = "terraform-aws-misc" - version = trimspace(file("${path.module}/../../VERSION")) - module = basename(path.module) - name = var.name - } - module_tags = var.module_tags_enabled ? { - "module.terraform.io/package" = local.metadata.package - "module.terraform.io/version" = local.metadata.version - "module.terraform.io/name" = local.metadata.module - "module.terraform.io/full-name" = "${local.metadata.package}/${local.metadata.module}" - "module.terraform.io/instance" = local.metadata.name - } : {} -} - - -################################################### -# Configuration for MSK Cluster -################################################### - -locals { - server_properties = < 0 ? 1 : 0 - - cluster_arn = aws_msk_cluster.this.arn - secret_arn_list = values(module.secret).*.arn -} diff --git a/modules/msk-cluster/security-group.tf b/modules/msk-cluster/security-group.tf deleted file mode 100644 index be49bf1..0000000 --- a/modules/msk-cluster/security-group.tf +++ /dev/null @@ -1,124 +0,0 @@ -data "aws_subnet" "this" { - id = var.broker_subnets[0] -} - -locals { - vpc_id = data.aws_subnet.this.vpc_id -} - - -################################################### -# Security Group -################################################### - -module "security_group" { - source = "tedilabs/network/aws//modules/security-group" - version = "0.26.0" - - count = length(var.broker_allowed_ingress_cidrs) > 0 ? 1 : 0 - - name = var.name - description = "Security group for MSK Cluster." - vpc_id = local.vpc_id - - ingress_rules = [ - { - id = "broker-plaintext/cidrs" - description = "Allow CIDRs to communicate with Kafka brokers in plaintext." - protocol = "tcp" - from_port = 9092 - to_port = 9092 - - cidr_blocks = var.broker_allowed_ingress_cidrs - }, - { - id = "broker-tls/cidrs" - description = "Allow CIDRs to communicate with Kafka brokers in tls." - protocol = "tcp" - from_port = 9094 - to_port = 9094 - - cidr_blocks = var.broker_allowed_ingress_cidrs - }, - { - id = "broker-sasl-scram/cidrs" - description = "Allow CIDRs to communicate with Kafka brokers in SASL SCRAM." - protocol = "tcp" - from_port = 9096 - to_port = 9096 - - cidr_blocks = var.broker_allowed_ingress_cidrs - }, - { - id = "broker-sasl-iam/cidrs" - description = "Allow CIDRs to communicate with Kafka brokers in SASL IAM." - protocol = "tcp" - from_port = 9098 - to_port = 9098 - - cidr_blocks = var.broker_allowed_ingress_cidrs - }, - { - id = "broker-public-tls/cidrs" - description = "Allow CIDRs to communicate with Kafka brokers in tls (public)." - protocol = "tcp" - from_port = 9194 - to_port = 9194 - - cidr_blocks = var.broker_allowed_ingress_cidrs - }, - { - id = "broker-public-sasl-scram/cidrs" - description = "Allow CIDRs to communicate with Kafka brokers in SASL SCRAM (public)." - protocol = "tcp" - from_port = 9196 - to_port = 9196 - - cidr_blocks = var.broker_allowed_ingress_cidrs - }, - { - id = "broker-public-sasl-iam/cidrs" - description = "Allow CIDRs to communicate with Kafka brokers in SASL IAM (public)." - protocol = "tcp" - from_port = 9198 - to_port = 9198 - - cidr_blocks = var.broker_allowed_ingress_cidrs - }, - { - id = "zookeeper/cidrs" - description = "Allow CIDRs to communicate with Kafka zookeepers." - protocol = "tcp" - from_port = 2181 - to_port = 2181 - - cidr_blocks = var.broker_allowed_ingress_cidrs - }, - { - id = "prometheus-jmx-exporter/cidrs" - description = "Allow CIDRs to communicate with Prometheus JMX Exporter." - protocol = "tcp" - from_port = 11001 - to_port = 11001 - - cidr_blocks = var.broker_allowed_ingress_cidrs - }, - { - id = "prometheus-node-exporter/cidrs" - description = "Allow CIDRs to communicate with Prometheus Node Exporter." - protocol = "tcp" - from_port = 11002 - to_port = 11002 - - cidr_blocks = var.broker_allowed_ingress_cidrs - }, - ] - - resource_group_enabled = false - module_tags_enabled = false - - tags = merge( - local.module_tags, - var.tags, - ) -} diff --git a/modules/msk-cluster/variables.tf b/modules/msk-cluster/variables.tf deleted file mode 100644 index 634de59..0000000 --- a/modules/msk-cluster/variables.tf +++ /dev/null @@ -1,278 +0,0 @@ -variable "name" { - description = "(Required) Name of the MSK cluster." - type = string -} - -variable "kafka_version" { - description = "(Optional) Kafka version to use for the MSK cluster." - type = string - default = "2.8.0" - nullable = false -} - -variable "kafka_server_properties" { - description = "(Optional) Contents of the `server.properties` file for configuration of Kafka." - type = map(string) - default = {} - nullable = false -} - -variable "broker_size" { - description = "(Required) The desired total number of broker nodes in the kafka cluster. It must be a multiple of the number of specified client subnets." - type = number -} - -variable "broker_instance_type" { - description = "(Optional) The instance type to use for the kafka brokers." - type = string - default = "kafka.m5.large" - nullable = false -} - -variable "broker_volume_size" { - description = "(Optional) The size in GiB of the EBS volume for the data drive on each broker node. Minimum value of `1` and maximum value of `16384`. Defaults to `1000`." - type = number - default = 1000 - nullable = false - - validation { - condition = alltrue([ - var.broker_volume_size >= 1, - var.broker_volume_size <= 16384, - ]) - error_message = "Valid value for `broker_volume_size` is between `1` and `16384`." - } -} - -variable "broker_volume_provisioned_throughput_enabled" { - description = "(Optional) Whether provisioned throughput is enabled or not. You can specify the provisioned throughput rate in MiB per second for clusters whose brokers are of type `kafka.m5.4xlarge` or larger and if the storage volume is 10 GiB or greater. Defaults to `false`." - type = bool - default = false - nullable = false -} - -variable "broker_volume_provisioned_throughput" { - description = "(Optional) Throughput value of the EBS volumes for the data drive on each kafka broker node in MiB per second. The minimum value is `250`. The maximum value varies between broker type." - type = number - default = null -} - -variable "broker_subnets" { - description = "(Required) A list of subnet IDs to place ENIs of the MSK cluster broker nodes within." - type = list(string) -} - -variable "broker_public_access_enabled" { - description = "(Optional) Whether to allow public access to MSK brokers." - type = bool - default = false - nullable = false -} - -variable "broker_allowed_ingress_cidrs" { - description = "(Optional) A list of CIDR for MSK ingress access." - type = list(string) - default = [] -} - -variable "broker_additional_security_groups" { - description = "(Optional) A list of security group IDs to associate with ENIs to control who can communicate with the cluster." - type = list(string) - default = [] -} - -variable "auth_unauthenticated_access_enabled" { - description = "(Optional) Enables unauthenticated access. Defaults to `true`." - type = bool - default = true - nullable = false -} - -variable "auth_sasl_iam_enabled" { - description = "(Optional) Enables IAM client authentication." - type = bool - default = false - nullable = false -} - -variable "auth_sasl_scram_enabled" { - description = "(Optional) Enables SCRAM client authentication via AWS Secrets Manager." - type = bool - default = false - nullable = false -} - -variable "auth_sasl_scram_kms_key" { - description = "(Optional) The ARN of a KMS key to encrypt AWS SeecretsManager Secret resources for storing SASL/SCRAM authentication data. Only required when the MSK cluster has SASL/SCRAM authentication enabled. The Username/Password Authentication based on SASL/SCRAM needs to create a Secret resource in AWS SecretsManager with a custom AWS KMS Key. A secret created with the default AWS KMS key cannot be used with an Amazon MSK cluster." - type = string - default = null -} - -variable "auth_sasl_scram_users" { - description = "(Optional) A list of usernames to be allowed for SASL/SCRAM authentication to the MSK cluster. The password for each username is randomly generated and stored in AWS SecretsManager secret." - type = set(string) - default = [] - nullable = false -} - -variable "auth_tls_enabled" { - description = "(Optional) Enables TLS client authentication." - type = bool - default = false - nullable = false -} - -variable "auth_tls_acm_ca_arns" { - description = "(Optional) List of ACM Certificate Authority Amazon Resource Names (ARNs)." - type = list(string) - default = [] - nullable = false -} - -variable "encryption_at_rest_kms_key" { - description = "(Optional) Specify a KMS key short ID or ARN (it will always output an ARN) to use for encrypting your data at rest. If no key is specified, an AWS managed KMS ('aws/msk' managed service) key will be used for encrypting the data at rest." - type = string - default = "" -} - -variable "encryption_in_transit_in_cluster_enabled" { - description = "(Optional) Whether data communication among broker nodes is encrypted." - type = bool - default = true - nullable = false -} - -variable "encryption_in_transit_client_mode" { - description = "(Optional) Encryption setting for data in transit between clients and brokers. `TLS`, `TLS_PLAINTEXT`, `PLAINTEXT` are available." - type = string - default = "TLS_PLAINTEXT" - nullable = false - - validation { - condition = contains(["TLS", "TLS_PLAINTEXT", "PLAINTEXT"], var.encryption_in_transit_client_mode) - error_message = "Valid values are `TLS`, `TLS_PLAINTEXT`, `PLAINTEXT`." - } -} - -variable "logging_cloudwatch_enabled" { - description = "(Optional) Indicates whether you want to enable or disable streaming broker logs to Cloudwatch Logs." - type = bool - default = false - nullable = false -} - -variable "logging_cloudwatch_log_group" { - description = "(Optional) The name of log group on CloudWatch Logs to deliver logs to." - type = string - default = "" - nullable = false -} - -variable "logging_firehose_enabled" { - description = "(Optional) Indicates whether you want to enable or disable streaming broker logs to Kinesis Data Firehose." - type = bool - default = false - nullable = false -} - -variable "logging_firehose_delivery_stream" { - description = "(Optional) Name of the Kinesis Data Firehose delivery stream to deliver logs to." - type = string - default = "" - nullable = false -} - -variable "logging_s3_enabled" { - description = "(Optional) Indicates whether you want to enable or disable streaming broker logs to S3." - type = bool - default = false - nullable = false -} - -variable "logging_s3_bucket" { - description = "(Optional) The name of the S3 bucket to deliver logs to." - type = string - default = "" - nullable = false -} - -variable "logging_s3_prefix" { - description = "(Optional) The prefix to append to the folder name." - type = string - default = "" - nullable = false -} - -variable "monitoring_cloudwatch_level" { - description = "(Optional) The desired enhanced MSK CloudWatch monitoring level. `DEFAULT`, `PER_BROKER`, `PER_TOPIC_PER_BROKER`, `PER_TOPIC_PER_PARTITION` are available." - type = string - default = "DEFAULT" - nullable = false - - validation { - condition = contains(["DEFAULT", "PER_BROKER", "PER_TOPIC_PER_BROKER", "PER_TOPIC_PER_PARTITION"], var.monitoring_cloudwatch_level) - error_message = "Valid values are `DEFAULT`, `PER_BROKER`, `PER_TOPIC_PER_BROKER`, `PER_TOPIC_PER_PARTITION`." - } -} - -variable "monitoring_prometheus_jmx_exporter_enabled" { - description = "(Optional) Indicates whether you want to enable or disable the JMX Exporter." - type = bool - default = false - nullable = false -} - -variable "monitoring_prometheus_node_exporter_enabled" { - description = "(Optional) Indicates whether you want to enable or disable the Node Exporter." - type = bool - default = false - nullable = false -} - -variable "timeouts" { - description = "(Optional) How long to wait for the MSK cluster to be created/updated/deleted." - type = map(string) - default = { - create = "120m" - update = "120m" - delete = "120m" - } - nullable = false -} - -variable "tags" { - description = "(Optional) A map of tags to add to all resources." - type = map(string) - default = {} - nullable = false -} - -variable "module_tags_enabled" { - description = "(Optional) Whether to create AWS Resource Tags for the module informations." - type = bool - default = true - nullable = false -} - - -################################################### -# Resource Group -################################################### - -variable "resource_group_enabled" { - description = "(Optional) Whether to create Resource Group to find and group AWS resources which are created by this module." - type = bool - default = true -} - -variable "resource_group_name" { - description = "(Optional) The name of Resource Group. A Resource Group name can have a maximum of 127 characters, including letters, numbers, hyphens, dots, and underscores. The name cannot start with `AWS` or `aws`." - type = string - default = "" -} - -variable "resource_group_description" { - description = "(Optional) The description of Resource Group." - type = string - default = "Managed by Terraform." -} diff --git a/modules/msk-cluster/versions.tf b/modules/msk-cluster/versions.tf deleted file mode 100644 index 583387c..0000000 --- a/modules/msk-cluster/versions.tf +++ /dev/null @@ -1,14 +0,0 @@ -terraform { - required_version = ">= 1.2" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 4.22" - } - random = { - source = "hashicorp/random" - version = ">= 3.3" - } - } -} From 99a48966be0866dc898a950663b70df427aabc5f Mon Sep 17 00:00:00 2001 From: "Byungjin Park (Claud)" Date: Sun, 3 Sep 2023 00:25:06 +0900 Subject: [PATCH 11/12] Update resource-group module to support optional attributes (#35) --- modules/resource-group/README.md | 6 +++--- modules/resource-group/main.tf | 4 ++-- modules/resource-group/outputs.tf | 4 ++-- modules/resource-group/variables.tf | 14 +++++++++++--- modules/resource-group/versions.tf | 2 +- 5 files changed, 19 insertions(+), 11 deletions(-) diff --git a/modules/resource-group/README.md b/modules/resource-group/README.md index 348fae5..5cb4712 100644 --- a/modules/resource-group/README.md +++ b/modules/resource-group/README.md @@ -9,14 +9,14 @@ This module creates following resources. | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.1 | +| [terraform](#requirement\_terraform) | >= 1.5 | | [aws](#requirement\_aws) | >= 4.14 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | 4.16.0 | +| [aws](#provider\_aws) | 5.15.0 | ## Modules @@ -35,7 +35,7 @@ No modules. | [name](#input\_name) | (Required) A name to identify the resource group. A resource group name can have a maximum of 127 characters, including letters, numbers, hyphens, dots, and underscores. The name cannot start with `AWS` or `aws`. | `string` | n/a | yes | | [description](#input\_description) | (Optional) The description of the resource group. | `string` | `"Managed by Terraform."` | no | | [module\_tags\_enabled](#input\_module\_tags\_enabled) | (Optional) Whether to create AWS Resource Tags for the module informations. | `bool` | `true` | no | -| [query](#input\_query) | (Optional) A configuration for the actual query used to match against resources. It supports `resource_types` and `resource_tags`. `query` block as defined below.
(Required) `resource_tags` - A map of key/value pairs that are compared to the tags attached to resources.
(Optional) `resource_types` - A list of resource-type specification strings with `AWS::service-id::resource-type` format. Limit the results to only those resource types that match the filter. Specify `AWS::AllSupported` to include resources of any resources that are currently supported by Resource Group. | `any` | `{}` | no | +| [query](#input\_query) | (Optional) A configuration for the actual query used to match against resources. It supports `resource_types` and `resource_tags`. `query` block as defined below.
(Optional) `resource_tags` - A map of key/value pairs that are compared to the tags attached to resources.
(Optional) `resource_types` - A list of resource-type specification strings with `AWS::service-id::resource-type` format. Limit the results to only those resource types that match the filter. Specify `AWS::AllSupported` to include resources of any resources that are currently supported by Resource Group. | 
object({
    resource_tags  = optional(map(string), {})
    resource_types = optional(list(string), ["AWS::AllSupported"])
  })
| `{}` | no | | [tags](#input\_tags) | (Optional) A map of tags to add to all resources. | `map(string)` | `{}` | no | ## Outputs diff --git a/modules/resource-group/main.tf b/modules/resource-group/main.tf index 1efd8b3..fe9e009 100644 --- a/modules/resource-group/main.tf +++ b/modules/resource-group/main.tf @@ -21,14 +21,14 @@ locals { locals { filters = [ - for key, value in try(var.query.resource_tags, {}) : { + for key, value in var.query.resource_tags : { "Key" = key "Values" = flatten([value]) } ] query = <<-JSON { - "ResourceTypeFilters": ${jsonencode(try(var.query.resource_types, ["AWS::AllSupported"]))}, + "ResourceTypeFilters": ${jsonencode(var.query.resource_types)}, "TagFilters": ${jsonencode(local.filters)} } JSON diff --git a/modules/resource-group/outputs.tf b/modules/resource-group/outputs.tf index 1e0f6ea..4ddf8a8 100644 --- a/modules/resource-group/outputs.tf +++ b/modules/resource-group/outputs.tf @@ -15,10 +15,10 @@ output "description" { output "resource_types" { description = "The resource types used by the resource group to query resources." - value = try(var.query.resource_types, ["AWS::AllSupported"]) + value = var.query.resource_types } output "resource_tags" { description = "The resource tags used by the resource group to query resources." - value = try(var.query.resource_tags, {}) + value = var.query.resource_tags } diff --git a/modules/resource-group/variables.tf b/modules/resource-group/variables.tf index 6ae4201..70b82c2 100644 --- a/modules/resource-group/variables.tf +++ b/modules/resource-group/variables.tf @@ -1,32 +1,40 @@ variable "name" { description = "(Required) A name to identify the resource group. A resource group name can have a maximum of 127 characters, including letters, numbers, hyphens, dots, and underscores. The name cannot start with `AWS` or `aws`." type = string + nullable = false } variable "description" { description = "(Optional) The description of the resource group." type = string default = "Managed by Terraform." + nullable = false } variable "query" { description = < Date: Sun, 3 Sep 2023 00:25:30 +0900 Subject: [PATCH 12/12] Bump to v0.11.0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 5712157..d9df1bb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.10.1 +0.11.0