Rackspace Cloud Files - purge by container with Ruby Fog gem

rackdel.rb

#!/bin/env ruby
# Last used with 3.2.2 - purge a Rackspace cloud files container.
# Put fog-rackspace in your gemfile and run: bundle install && bundle exec ./rackdel.rb <container name>
# limits don't seem to quite work correctly since this will empty your container even past 10_000 items per container?? 
# oO Definately could be improved.

require 'net/http'
require 'fog/rackspace'

CONNECTION_OPTIONS = {
  provider: 'Rackspace',
  rackspace_region: 'DFW',
  rackspace_username: '',
  rackspace_api_key: ''
}

$container = ARGV[0]
$running = true
$limit_threads = false
$markers = { 0 => '' }

def inner_loop ndx=0, limit=10000
  while $markers[ndx].nil? && !$limit_threads && $running
    sleep 1
  end
  return nil if $limit_threads || !$running
  return Thread.new do
    storage = Fog::Storage.new(CONNECTION_OPTIONS)         
    
    d = storage.directories.select {|x| x.key == $container}.first
    dir = (storage.directories.get d.key, {limit:limit, marker:$markers[ndx]})     
    deletes = []
    files = dir.files
    puts "starting new thread with limit: #{limit}, marker: #{$markers[ndx]} and #{files.count} files in the set"
    if files.count > 0 && files.count >= 10000
      $markers[ndx+1] = files.last.key
    else 
      $limit_threads = true
    end    
    cycle = 0
    while $running
      break unless files.count > 0
      files.each do |f|    
        deletes << f.key
        if deletes.count >= 2500          
          deletes.clear unless purge_items( deletes, storage, {thread: ndx, cycle: cycle } ) <= 0
          cycle += 1
        end    
      end
      deletes.clear unless purge_items( deletes, storage, {thread: ndx, cycle: cycle } ) <= 0
      break
    end
    puts "Exiting thread #{ndx}"
  end  
end

def purge_items deletes, connection, options={}
  c = deletes.count
  puts "DELETING BATCH ************************** thread: #{options[:thread]} cycle: #{options[:cycle]}, count #{deletes.count}, starting with #{deletes.first}"
  ret = connection.delete_multiple_objects($container,deletes)
  if ret.status > 299 || ret.body["Errors"].size > 0
    puts ret.inspect
    exit(-2)
  end
  return c
rescue Excon::Error::Timeout => e
  puts  ">>>> ERROR - Retry: #{e.message}\n"
  return 0  
rescue StandardError => e
  puts  ">>>> Fatal ERROR: #{e.message}\n"
  e.backtrace.each {|line| puts "#{line}\n"}
  @running = false
  exit(-1)
end


threads = []
0.step( to:11 ) do |i|
  new_thread = inner_loop i
  threads << new_thread unless new_thread.nil?    
end

storage = Fog::Storage.new(CONNECTION_OPTIONS)             
while $running
  sleep 20
  d = storage.directories.select {|x| x.key == $container}.first
  dir = (storage.directories.get d.key)  
  puts "Stat update - File Count: #{dir.count}, Size: #{dir.bytes}"
  break unless dir.count > 0
end
threads.each do |t|
  t.join
end