mount-s3 auto mount with systemd

mount-s3 auto mount with systemd

# mount-s3 is AWS native solution to mount S3 to EC2 without the need of SGW. Downside is you don't get the performance, but still have to pay for the s3 api calls.
# Reference docs 
# https://aws.amazon.com/blogs/aws/mountpoint-for-amazon-s3-generally-available-and-ready-for-production-workloads/
# https://github.com/awslabs/mountpoint-s3
# As replacement of https://github.com/s3fs-fuse/s3fs-fuse mount-s3 is missing a key feature to auto mount bucket on reboot.

# The following instructions will help you automate mounting of S3 to EC2 as persistent storage using systemd.

# Create and Attach IAM policy to instance profile role (replace resource ARN). Name of policy S3-ReadWrite-<bucketname>

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "ListObjectsInBucket",
            "Effect": "Allow",
            "Action": [
                "s3:ListBucket"
            ],
            "Resource": [
                "arn:aws:s3:::<bucketname>"
            ]
        },
        {
            "Sid": "AllObjectActions",
            "Effect": "Allow",
            "Action": "s3:*Object",
            "Resource": [
                "arn:aws:s3:::<bucketname>/*"
            ]
        }
    ]
}

# Install (this example is for Ubuntu, use rpm or other package manager, depending on your destro)

wget https://s3.amazonaws.com/mountpoint-s3-release/latest/x86_64/mount-s3.deb
sudo apt-get install -y ./mount-s3.deb

# Create directory for mount point

sudo mkdir -p <path> 

# Create bash script mount.s3.sh

sudo vi /usr/local/bin/mount-s3.sh

#!/bin/bash
sleep 10
mount-s3 <bucketname> <path> --allow-other

# Make script executable

chmod u+x /usr/local/bin/mount-s3.sh

# Create config file in systemd for the service that starts the script 

sudo vi /lib/systemd/system/mount-s3.service

[Unit]
Description=Auto mount s3
Wants=network-online.target
After=network-online.target

[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/bin/sh /usr/local/bin/mount-s3.sh

[Install]
WantedBy=default.target

# Enable systemd on reboot, start service

systemctl enable mount-s3.service
systemctl start mount-s3.service
systemctl status mount-s3.service # must be active / loaded & mountpoint show S3 contents


============== Cloudformation ==============

Metadata:
  AWS::CloudFormation::Interface:
    ParameterGroups:
    - Label:
        default: Instance configuration
      Parameters:
      - RDSexportBucket
      - s3mountpath
    ParameterLabels:
      RDSexportBucket:
        default: S3 bucket to mount to EC2
Parameters:
  RDSexportBucket:
    AllowedPattern: ^[0-9a-zA-Z]?([0-9a-zA-Z-]*[0-9a-zA-Z])*$
    ConstraintDescription: S3 bucket name can include numbers, lowercase letters,
      uppercase letters, and hyphens (-). It cannot start or end with a hyphen (-).
    Description: S3 bucket name . The bucket is then mounted on EC2 to run queries.
Resources:
  InstanceRole:
    Type: AWS::IAM::Role
    Properties:
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM
        - arn:aws:iam::aws:policy/AmazonEC2ReadOnlyAccess
      AssumeRolePolicyDocument:
        Statement:
        - Effect: Allow
          Principal:
            Service:
            - ec2.amazonaws.com
          Action:
          - sts:AssumeRole
      Path: /

  InstanceRolePolicy:
    Type: AWS::IAM::Policy
    Properties:
      PolicyName: InstancePolicy
      PolicyDocument:
        Version: 2012-10-17
        Statement:
        - Effect: Allow
          Action: s3:ListBucket
          Resource: !Sub arn:aws:s3:::${RDSexportBucket}
        - Effect: Allow
          Action: s3:*Object
          Resource: !Sub arn:aws:s3:::${RDSexportBucket}/*
      Roles:
      - !Ref InstanceRole

  InstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Path: /
      Roles:
      - !Ref InstanceRole


Metadata:
  AWS::CloudFormation::Interface:
    ParameterGroups:
    - Label:
        default: Instance configuration
      Parameters:
      - RDSexportBucket
      - s3mountpath
    ParameterLabels:
      RDSexportBucket:
        default: S3 bucket to mount to EC2
Parameters:
  RDSexportBucket:
    AllowedPattern: ^[0-9a-zA-Z]?([0-9a-zA-Z-]*[0-9a-zA-Z])*$
    ConstraintDescription: S3 bucket name can include numbers, lowercase letters,
      uppercase letters, and hyphens (-). It cannot start or end with a hyphen (-).
    Description: S3 bucket name . The bucket is then mounted on EC2 to run queries.
  s3mountpath:
    AllowedPattern: /[/a-zA-Z0-9_\\-\\.]*
    ConstraintDescription: Can include lowercase letters, uppercase letters, numbers,
      hyphens, forward slash.
    Description: Path for S3 mount point 
Resources:
  InstanceRole:
    Type: AWS::IAM::Role
    Properties:
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM
        - arn:aws:iam::aws:policy/AmazonEC2ReadOnlyAccess
      AssumeRolePolicyDocument:
        Statement:
        - Effect: Allow
          Principal:
            Service:
            - ec2.amazonaws.com
          Action:
          - sts:AssumeRole
      Path: /

  InstanceRolePolicy:
    Type: AWS::IAM::Policy
    Properties:
      PolicyName: InstancePolicy
      PolicyDocument:
        Version: 2012-10-17
        Statement:
        - Effect: Allow
          Action: s3:ListBucket
          Resource: !Sub arn:aws:s3:::${RDSexportBucket}
        - Effect: Allow
          Action: s3:*Object
          Resource: !Sub arn:aws:s3:::${RDSexportBucket}/*
      Roles:
      - !Ref InstanceRole

  InstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Path: /
      Roles:
      - !Ref InstanceRole    

      IamInstanceProfile: !Ref InstanceProfile
      UserData:
        Fn::Base64:
              Fn::Sub: |
                  #!/bin/bash
                  # Automate mount point of s3 https://gist.github.com/didorins/1e73318e1f6d34ff8a983b90cbe99bd1
                  wget https://s3.amazonaws.com/mountpoint-s3-release/latest/x86_64/mount-s3.deb
                  apt-get install -y ./mount-s3.deb
                  mkdir -p ${s3mountpath}
                  cat <<EOT > /usr/local/bin/mount-s3.sh
                  #!/bin/bash
                  sleep 10
                  mount-s3 ${RDSexportBucket} ${s3mountpath} --allow-other
                  EOT
                  chmod u+x /usr/local/bin/mount-s3.sh
                  cat <<EOT > /lib/systemd/system/mount-s3.service
                  [Unit]
                  Description=Auto mount s3
                  Wants=network-online.target
                  After=network-online.target
                  [Service]
                  Type=oneshot
                  RemainAfterExit=yes
                  ExecStart=/bin/sh /usr/local/bin/mount-s3.sh
                  [Install]
                  WantedBy=default.target
                  EOT
                  systemctl enable mount-s3.service
                  systemctl start mount-s3.service

============== Terraform ==============

locals {
  app_user_data = <<-EOF
  
    # Install and configure mount-s3 for S3 mount on EC2

    wget https://s3.amazonaws.com/mountpoint-s3-release/latest/x86_64/mount-s3.rpm
    yum install ./mount-s3.rpm -y
    mkdir -p ${var.path}
    cat <<EOT > /usr/local/bin/mount-s3.sh
    #!/bin/bash
    sleep 10
    mount-s3 ets-${var.env}-data ${var.path} --allow-other
    EOT
    chmod u+x /usr/local/bin/mount-s3.sh
    cat <<EOT > /lib/systemd/system/mount-s3.service
    [Unit]
    Description=Auto mount s3
    Wants=network-online.target
    After=network-online.target
    [Service]
    Type=oneshot
    RemainAfterExit=yes
    ExecStart=/bin/sh /usr/local/bin/mount-s3.sh
    [Install]
    WantedBy=default.target
    EOT
    systemctl enable mount-s3.service
    systemctl start mount-s3.service
    
      EOF
}

resource "aws_instance" "this" {

...
  user_data = local.app_user_data
}

variable "env" {
  description = "Name of environment"
}

variable "path" {
  description = "Mountpoint path for S3"
}