The Goal is to use Route53 for DNS resolution in an multi-account setup and also with an on-premise infrastructure.
AWS have a good article Simplify DNS management in a multi-account environment with Route 53 Resolver that I have implemented using Terraform in my Lab environment.
The main files in this Lab are :
This article explains things well. I wont try and explain it again, so you need to read this article first !
However the article does not have a working example. The following is the key terraform resources that you will need.
Firstly I assume you have registered you domain example.com in Route53 (or another registrar).
The "backbone" account is probably your AWS root account. It contains the Transit Gateway with attached Site-to-Site VPN (or ClientVPN). It has a VPC with 2 private subnets (multiple AZs). Public subnets should not be necessary for our purposes. It will also contain the Route53 EndPoints.
If you registered the domain in Route 53 you can import the Public Hosted Zone that it creates into your Terraform or delete and create as below :
# Imported the automatically created Zone (upon domain registration) using terraform e.g. :
# terraform-v1.0.11 import aws_route53_zone.public Z05031252J93KZFK3MNHW
resource "aws_route53_zone" "public" {
name = "example.com"
comment = "HostedZone created by Route53 Registrar"
force_destroy = false
}Create the Private Hosted Zone for our DNS sub-domain for the backbone account :
resource "aws_route53_zone" "private" {
name = "backbone.example.com"
comment = "Private zone for our VPC"
force_destroy = true
# Note: without specifying the vpc association here (below), we get a Public Zone (but wanted a Private one)
# Hence could not use the resource "aws_route53_zone_association" for this association
vpc {
vpc_id = module.vpc.vpc_id
}
}Create the Route53 Inbound Endpoint and its Security Group :
resource "aws_route53_resolver_endpoint" "dns-endpoint-inbound" {
name = "route53-dns-endpoint-inbound"
direction = "INBOUND"
security_group_ids = [
aws_security_group.dns-endpoint-inbound.id
]
dynamic "ip_address" {
for_each = module.vpc.private_subnets
content {
subnet_id = ip_address.value
}
}
}
resource "aws_security_group" "dns-endpoint-inbound" {
name = "route53-dns-endpoint-inbound"
description = "Access to Route53 DNS Resolver Endpoint (Inbound)"
vpc_id = module.vpc.vpc_id
tags = merge(module.global_variables.default_tags, var.environment.default_tags, {
Name = "route53-dns-endpoint-inbound"
})
}
# All ingress to port 53
resource "aws_security_group_rule" "dns-endpoint-inbound-allow-ingress-all" {
type = "ingress"
description = "DNS"
from_port = 53
to_port = 53
protocol = "all"
cidr_blocks = concat(
"10.1.0.0/16", # backbone vpc
"10.2.0.0/16", # core vpc
"10.3.0.0/16", # VPN clients CIDR
)
security_group_id = aws_security_group.dns-endpoint-inbound.id
}
# All egress to port 53
resource "aws_security_group_rule" "dns-endpoint-inbound-allow-egress-all" {
type = "egress"
description = "DNS"
from_port = 53
to_port = 53
protocol = "all"
cidr_blocks = concat(
"10.1.0.0/16", # backbone vpc
"10.2.0.0/16", # core vpc
"10.3.0.0/16", # VPN clients CIDR
)
security_group_id = aws_security_group.dns-endpoint-inbound.id
}and the Outbound Endpoint :
resource "aws_route53_resolver_endpoint" "dns-endpoint-outbound" {
name = "route53-dns-endpoint-outbound"
direction = "OUTBOUND"
security_group_ids = [
aws_security_group.dns-endpoint-outbound.id
]
dynamic "ip_address" {
for_each = module.vpc.private_subnets
content {
subnet_id = ip_address.value
}
}
}
resource "aws_security_group" "dns-endpoint-outbound" {
name = "route53-dns-endpoint-outbound"
description = "Access to Route53 DNS Resolver Endpoint (Outbound)"
vpc_id = module.vpc.vpc_id
tags = merge(module.global_variables.default_tags, var.environment.default_tags, {
Name = "route53-dns-endpoint-outbound"
})
}
resource "aws_security_group_rule" "dns-endpoint-outbound-allow-ingress-all" {
type = "ingress"
description = "DNS"
from_port = 53
to_port = 53
protocol = "all"
cidr_blocks = concat(
"10.1.0.0/16", # backbone vpc
"10.2.0.0/16", # core vpc
)
security_group_id = aws_security_group.dns-endpoint-outbound.id
}
resource "aws_security_group_rule" "dns-endpoint-outbound-allow-egress-all" {
type = "egress"
description = "DNS"
from_port = 53
to_port = 53
protocol = "all"
cidr_blocks = concat(
"10.1.0.0/16", # backbone vpc
"10.2.0.0/16", # core vpc
"10.3.0.0/16", # VPN clients CIDR
)
security_group_id = aws_security_group.dns-endpoint-outbound.id
}And the Route53 forwarding rules :
# Inbound Endpoint Forward "example.com" requests to Backbone DNS server
resource "aws_route53_resolver_rule" "aws-cloud" {
domain_name = "example.com"
name = "aws-cloud"
rule_type = "FORWARD"
resolver_endpoint_id = aws_route53_resolver_endpoint.dns-endpoint-outbound.id
dynamic "target_ip" {
for_each = aws_route53_resolver_endpoint.dns-endpoint-inbound.ip_address
content {
ip = target_ip.value.ip
}
}
}
# Outbound EndPoint Forward the on-premise domain names requests to our on-premise DNS server
resource "aws_route53_resolver_rule" "on-premise" {
domain_name = "on-premise.com"
name = "on-premise"
rule_type = "FORWARD"
resolver_endpoint_id = aws_route53_resolver_endpoint.dns-endpoint-outbound.id
target_ip {
ip = "192.168.1.10" # our on-premise DNS server
}
}
resource "aws_route53_resolver_rule_association" "on-premise" {
name = "route53-dns-endpoint-outbound"
resolver_rule_id = aws_route53_resolver_rule.on-premise.id
vpc_id = module.vpc.vpc_id
}Share the "aws-cloud" resolver rule with your other accounts :
# Note: assumes that "Sharing within the AWS Organisation" is ENABLED.
# https://docs.aws.amazon.com/ram/latest/userguide/getting-started-sharing.html#getting-started-sharing-orgs
locals {
share_with_account_ids = [
"111111111111" # your core account id
# .... can add others here
]
}
resource "aws_ram_resource_share" "route53-resolver-rule-aws-cloud" {
name = "resolver-rule-aws-cloud-share-aws-cloud"
allow_external_principals = false
tags = merge(var.default_tags, {
Name = var.name
})
}
resource "aws_ram_resource_association" "route53-resolver-rule-aws-cloud" {
resource_arn = aws_route53_resolver_rule.aws-cloud.arn
resource_share_arn = aws_ram_resource_share.route53-resolver-rule-aws-cloud.arn
}
resource "aws_ram_principal_association" "route53-resolver-rule-aws-cloud" {
count = length(local.share_with_account_ids)
principal = local.share_with_account_ids[count.index]
resource_share_arn = aws_ram_resource_share.route53-resolver-rule-aws-cloud.arn
}And share the "on-premise" resolver rule with your other accounts :
# Note: assumes that "Sharing within the AWS Organisation" is ENABLED.
# https://docs.aws.amazon.com/ram/latest/userguide/getting-started-sharing.html#getting-started-sharing-orgs
locals {
share_with_account_ids = [
"111111111111" # your core account id
# .... can add others here
]
}
resource "aws_ram_resource_share" "route53-resolver-rule-on-premise" {
name = "resolver-rule-aws-cloud-share-on-premise"
allow_external_principals = false
tags = merge(var.default_tags, {
Name = var.name
})
}
resource "aws_ram_resource_association" "route53-resolver-rule-on-premise" {
resource_arn = aws_route53_resolver_rule.on-premise.arn
resource_share_arn = aws_ram_resource_share.route53-resolver-rule-on-premise.arn
}
resource "aws_ram_principal_association" "route53-resolver-rule-on-premise" {
count = length(local.share_with_account_ids)
principal = local.share_with_account_ids[count.index]
resource_share_arn = aws_ram_resource_share.route53-resolver-rule-on-premise.arn
}and lastly we want to associate our VPC with the other accounts Private Hosted Zones.
However we havent terraformed the other accounts yet so we dont know their Private HostedZone IDs. To workaround this I use a "second phase" approach. I look for files named "terraform-output-route53-private-hosted-zone-id" (which wont exist yet and so we have "conditional" terraform). After core is terraformed we re-run backbone terraform.
# ---------------------------------------------------------------------------------------------------------------------
# SECOND PHASE : This needs to be run after a creation/change of one (or more) cross-account Private Hosted Zones (in other accounts)
# This Stage will attempt to associate the Backbone VPC with those cross-account Private Hosted Zones.
# Otherwise no DNS resolution from Backbone to cross-account hosted FQDNs
# ---------------------------------------------------------------------------------------------------------------------
locals {
route53_private_hosted_zone_id_filenames = fileset("${path.module}/..", "*/terraform-output-route53-private-hosted-zone-id")
other_account_private_hosted_zone_ids = [for file in data.local_file.route53_private_hosted_zone_id_files : file["content"]]
}
data "local_file" "route53_private_hosted_zone_id_files" {
for_each = local.route53_private_hosted_zone_id_filenames
filename = "${path.module}/../${each.value}"
}
resource "aws_route53_zone_association" "backbone-private-and-other-vpcs" {
count = length(local.other_account_private_hosted_zone_ids)
zone_id = local.other_account_private_hosted_zone_ids[count.index]
vpc_id = module.vpc.vpc_id
}
resource "local_file" "backbone_vpc_id" {
content = module.vpc.vpc_id
directory_permission = "660"
file_permission = "660"
filename = "${path.module}/terraform-output-vpc-id"
}Note in above we output the backbone VPC ID to file terraform-output-vpc-id using local_file resource. This will be read by core and is simpler than reading the backbone terraform state from S3 bucket (at least for small-scale sharing of data).
If using a Client VPN then use the Route53 Inbound Endpoint IP addresses, as below, so connected VPN clients have DNS resolution for our AWS resources :
resource "aws_ec2_client_vpn_endpoint" "client-vpn" {
description = "${var.environment.resource_name_prefix}-client-vpn"
server_certificate_arn = aws_acm_certificate.client-vpn-server-cert.arn
client_cidr_block = "10.3.0.0/16"
dns_servers = aws_route53_resolver_endpoint.dns-endpoint-inbound.ip_address[*].ip
split_tunnel = true
authentication_options {
type = "certificate-authentication"
root_certificate_chain_arn = aws_acm_certificate.client-vpn-server-cert.arn
}
connection_log_options {
enabled = true
cloudwatch_log_group = aws_cloudwatch_log_group.client-vpn.name
cloudwatch_log_stream = aws_cloudwatch_log_stream.client-vpn.name
}
tags = merge(module.global_variables.default_tags, {
Name = "client-vpn"
})
}We will create a Private Hosted Zone for our accounts DNS sub-domain core.example.com :
resource "aws_route53_zone" "private" {
name = "core.example.com"
comment = "Private zone for our VPC"
force_destroy = true
# Must specify the vpc association (below), to get a Private zone...
vpc {
vpc_id = module.vpc.vpc_id
}
# Prevent the deletion of associated Backbone VPC, after the initial creation.
# See documentation on aws_route53_zone_association for details :
# https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route53_zone_association
lifecycle {
ignore_changes = [vpc]
}
}We output our above Private Hosted Zone ID to file, so this can be read by the backbone terraform during its "second phase" run and can associate its VPC to the core Private Hosted Zone.
This could also be achieved by reading terraform state files but then get caught in a catch-22 circular-dependancy hell.
resource "local_file" "route53_private_hosted_zone_id" {
content = aws_route53_zone.private.id
directory_permission = "660"
file_permission = "660"
filename = "${path.module}/terraform-output-route53-private-hosted-zone-id"
}During the backbone terraform run it out its VPC ID to file backbone/terraform-output-vpc-id. Read this in now using local_file resource.
Note: we could get this data from backbone's terraform state file, but it would affect how you invoke this terraform (e.g. via aws-vault). Also the sharing of state files between accounts increases the potential security "blast-radius" during a breach and the Bucket IAM Policy for it is complicated.
data "local_file" "backbone_vpc_id_file" {
filename = "${path.module}/../backbone/terraform-output-vpc-id"
}Because the private hosted zone and DNS-VPC are in different accounts, we need to associate the private hosted zone with the backbone "DNS-VPC". To do that, you need to create authorization from the account that owns the private hosted zone and accept this authorization from the account that owns the DNS-VPC.
resource "aws_route53_vpc_association_authorization" "example" {
vpc_id = data.local_file.backbone_vpc_id_file.content
zone_id = aws_route53_zone.private.id
}The Route53 Resolver Rules were shared by the backbone account via AWS RAM. Look these up and associate them with our core VPC, below :
data "aws_route53_resolver_rule" "aws-cloud" {
name = "aws-cloud"
}
resource "aws_route53_resolver_rule_association" "aws-cloud" {
name = "${module.global_variables.org_short_name}-route53-dns-endpoint-inbound"
resolver_rule_id = data.aws_route53_resolver_rule.aws-cloud.id
vpc_id = module.vpc.vpc_id
}
data "aws_route53_resolver_rule" "on-premise" {
name = "on-premise"
}
resource "aws_route53_resolver_rule_association" "on-premise" {
name = "${module.global_variables.org_short_name}-route53-dns-endpoint-outbound"
resolver_rule_id = data.aws_route53_resolver_rule.on-premise.id
vpc_id = module.vpc.vpc_id
}To ensure our EC2 instances set their hostnames and register their DNS we use cloud-init.
For the EC2 User Data have a terraform template like this :
#cloud-config
preserve_hostname: false # Feels wrong setting this to false, but otherwise will
# preserve the aws internal hostname of "ip-99-99-99-99"
hostname: ${aws_ec2_instance_name}
fqdn: ${aws_ec2_instance_fqdn}
manage_etc_hosts: true
write_files:
- path: /usr/local/bin/cloud-init-runcmd.sh
permissions: '0700'
content: |
sudo yum -y install bind-utils unzip # Install bind-utils for dig and nslookup
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip awscliv2.zip
./aws/install
PRPL_ROUTE53_PRIVATE_HOSTED_ZONE_ID=${aws_route53_private_hosted_zone_id}
PRIVATE_IP_ADDRESS=$(ip route get 1 | awk '{print $NF;exit}')
HOSTNAME=$(hostname)
TTL="600"
# Now register our hostname with Route53 DNS server ...
aws route53 change-resource-record-sets --hosted-zone-id $${PRPL_ROUTE53_PRIVATE_HOSTED_ZONE_ID} --change-batch "{ \"Changes\": [ { \"Action\": \"UPSERT\", \"ResourceRecordSet\": { \"Name\": \"$${HOSTNAME}\", \"Type\": \"A\", \"TTL\": $${TTL}, \"ResourceRecords\": [ { \"Value\": \"$${PRIVATE_IP_ADDRESS}\" } ] } } ] }"
runcmd:
- /usr/local/bin/cloud-init-runcmd.shIn the AWS documentation it tends to be more focussed on setting the hostname with little mention of the need for an explicit aws cli call to register our DNS with our Route53 server as with the aws route53 change-resource-record-sets call. ** This is worth highlighting I think ! **
The EC2 instance (for testing our DNS) will need an IAM profile for registering its hostname on our Route53 DNS Private Hosted Zone :
resource "aws_iam_role" "test" {
name = "test"
max_session_duration = 43200
assume_role_policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Action": "sts:AssumeRole",
"Principal": {
"Service": "ec2.amazonaws.com"
},
"Effect": "Allow",
"Sid": ""
}
]
}
EOF
tags = merge(module.global_variables.default_tags, var.environment.default_tags, {
Name = "test"
})
}
resource "aws_iam_policy" "route53" {
name = "test-route53"
description = "Route53"
policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "Route53registerDNS",
"Action": [
"route53:ChangeResourceRecordSets",
"route53:GetHostedZone",
"route53:ListResourceRecordSets"
],
"Effect": "Allow",
"Resource": [
"arn:aws:route53:::hostedzone/${aws_route53_zone.private.id}"
]
}
]
}
EOF
}
resource "aws_iam_role_policy_attachment" "route53" {
role = aws_iam_role.test.name
policy_arn = aws_iam_policy.route53.arn
}
resource "aws_iam_instance_profile" "test" {
name = "test"
role = aws_iam_role.test.name
}and then create the CentOS 7 EC2 instance like :
data "aws_ami" "centos-7" {
most_recent = true
filter {
name = "product-code"
values = ["aw0evgkw8e5c1q413zgy5pjce"]
}
filter {
name = "virtualization-type"
values = ["hvm"]
}
owners = ["679593333241"] # Owner is CentOS.org
}
resource "aws_instance" "test" {
ami = data.aws_ami.centos-7.id
instance_type = "t3a.nano"
iam_instance_profile = aws_iam_instance_profile.test.name
subnet_id = module.vpc.private_subnets[0]
key_name = aws_key_pair.ssh.key_name
root_block_device {
delete_on_termination = true
encrypted = true
}
disable_api_termination = var.environment.resource_deletion_protection
user_data = templatefile("${path.module}/ec2-test-user-data.yaml", {
aws_ec2_instance_name = "test"
aws_ec2_instance_fqdn = "test.backbone.example.com"
aws_route53_private_hosted_zone_id = aws_route53_zone.private.id
})
tags = merge(module.global_variables.default_tags, var.environment.default_tags, {
Name = "test"
})
}With the above you should now have DNS resolution within an AWS account, across AWS accounts, and to/from on-premise.
It is worth noting that the Route53 Resolver Endpoints are not cheap at $0.125 per hour per Endpoint IP. You need at least 2 IPs per Endpoint. So this totals to 4 * $0.125 = $180 per month.
The terraform detailed in this article is an example. For the real-world terraform see my git repository.