drewyangdev

#!/bin/bash
export AWS_REGION=$1
export BUCKET_NAME=$2
export ORG_NAME=$3
export WORKFLOW_NAME=$4
export WORKER_CONTAINER_UID=$5
export WORKER_CONTAINER_GID=$6
echo "INFO :: Mount S3"
sed -i '/user_allow_other/s/^#//g' /etc/fuse.conf
[ ! -d "/mnt/s3/works/org/${ORG_NAME}/proj/${WORKFLOW_NAME}/inbox" ] && sudo mkdir -p /mnt/s3/works/org/${ORG_NAME}/proj/${WORKFLOW_NAME}/inbox

drewyangdev

labels:
  com.datadoghq.ad.checks: '{"docker": {"instances": [{"host": "%%host%%", "port": 8081}]}}'
  com.datadoghq.ad.tags: '["org:${ORG_NAME}", "project:${WORKFLOW_NAME}", "worker_type:${WORKER_DOCKER_SUBDIR}", "log_parser:${LOG_PARSER}"]'
  com.datadoghq.ad.logs: >-
    [{
      "type": "docker",
      "source": "${ORG_NAME}_${WORKFLOW_NAME}", 
      "service": "${WORKER_DOCKER_SUBDIR}_${APPLICATION}", 
      "log_processing_rules":[{
        "type": "multi_line",

drewyangdev

{
    "agent": {
        "metrics_collection_interval": 60,
        "run_as_user": "root"
    },
    "metrics": {
        "namespace": "$CONTRACT",
        "aggregation_dimensions": [["InstanceType"]],
        "append_dimensions": {
            "InstanceId": "${aws:InstanceId}",

drewyangdev

name: github_actions_contect_check

on: [pull_request]

jobs:
  context_check:
    runs-on: ubuntu-latest
    steps:
      - name: Dump GitHub context
        env:

drewyangdev

#!/bin/bash
## Recommend to mount a separate volume on /var/lib/docker to store all docker related stuff
sudo apt-get install -y \
      ca-certificates \
      curl \
      gnupg \
      lsb-release
sudo mkdir -p /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
echo \

drewyangdev

#!/bin/bash
## e.g sudo bash mount_efs.sh xxx.xxx.xxx.xxx /mnt/xxx
export EFS_IP=$1 # EFS network section, private IP within subnet
export MOUNT_DIR=$2
if [ ! -d "$MOUNT_DIR" ]; then
  mkdir -p $MOUNT_DIR
fi
mount -t nfs -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport $EFS_IP:/ $MOUNT_DIR
df -Th

drewyangdev

#!/bin/bash
sudo su
## 1. only debug runcmd
# edit /var/lib/cloud/instance/scripts/runcmd
cloud-init single --name scripts_user --frequency=always

## 2. re-generate runcmd from userdata
cloud-init single --name runcmd --frequency=always

drewyangdev

#!/bin/bash
# TODO - change nvidia driver installation input from DRIVER_VERSION to DRIVER_URL to fix "tesla" only
# TODO - add system user input to fix "ubuntu" only
############################################
## Nvidia driver version is determined by GPU architecture(instance type) and
## CUDA toolkit version that you want to use

## 1. Find the Nvidia architecture for your instance type: https://towardsdatascience.com/choosing-the-right-gpu-for-deep-learning-on-aws-d69c157d8c86

## Based on the Nvidia architecture,

drewyangdev

#!/bin/bash
## Use lsblk check device, usually e.g "/dev/xvdf", "/dev/nvme1n1"
## Partition will be "/dev/xvdf1", "/dev/nvme1n1p1"
## e.g sudo bash linux_gpt_volume.sh "/dev/nvme1n1" "/dev/nvme1n1p1" "/mnt/gpt-volume" 
export DEVICE=$1
export PARTITION=$2
export MOUNT_DIR=$3
parted $DEVICE --script mklabel gpt
parted $DEVICE --script mkpart primary 2048s 100%
mkfs -t ext4 $PARTITION

drewyangdev

import os
import sys
import io
import logging
import logging.handlers
from distutils.util import strtobool

# get logger level
DEBUG = strtobool(os.getenv("DEBUG", "True"))
# set logger config