huseinzol05

huseinzol05

"""
https://www.oumi.ai/docs/en/latest/_modules/oumi/core/callbacks/hf_mfu_callback.html
"""

import torch
import wandb
from transformers import TrainerCallback, TrainerState, TrainerControl

# Theoretical Peak Tensor Core Performance (BF16 / FP16)
DEVICES = {

huseinzol05

import os
import time
import ray
import torch
import torch.distributed as dist
from ray import train
from ray.train import ScalingConfig
from ray.train.torch import TorchTrainer, get_device

class RayConnection:

huseinzol05

huseinzol05

huseinzol05

import torch
import torch.nn.functional as F
from torch.nn.attention.flex_attention import flex_attention, create_block_mask
import torch

head_num = 16
dim = 128
seq_len = 100
chunk_size = 5
batch_size = 1

huseinzol05

# train_grpo.py
import re
import torch
from datasets import load_dataset, Dataset
from transformers import AutoTokenizer, AutoModelForCausalLM
from peft import LoraConfig
from trl import GRPOConfig, GRPOTrainer

# Load and prep dataset

huseinzol05

from glob import glob
from tqdm import tqdm
from multiprocess import Pool
import itertools
import zipfile
import os

def chunks(l, n):
    for i in range(0, len(l), n):
        yield (l[i: i + n], i // n)

huseinzol05

import json
import requests
import os
import boto3
import shutil
from tqdm import tqdm

client = boto3.client(
    'bedrock-runtime', 
    region_name='us-west-2',

huseinzol05

git clone https://github.com/nvidia/nccl-tests && cd nccl-tests
sudo apt-get install openmpi-bin openmpi-common libopenmpi-dev -y
make MPI=1 MPI_HOME=/usr/lib/x86_64-linux-gnu/openmpi
./build/all_reduce_perf -b 8 -e 128M -f 2 -g 4
mpirun -np 10 -H 10.224.0.47,10.224.0.70 ./build/all_reduce_perf -b 8 -e 128M -f 2 -g 4