get-first-addrs.js

#!/usr/bin/env node

/* 
* usage: RPC=... NR_ADDRS=10 node get-first-addrs.js
* 
* This script collects the first N unique addresses transacting on a chain, and saves them to a file.
* 
* Environment Variables:
* - (required) RPC: The RPC URL to connect to
* - (optional) NR_ADDRS: The number of unique addresses to collect (default: 10)
* - (optional) DEBUG: If set to 1, be more verbose
* - (optional) START_BLOCK: The block number to start collecting from (default: 1)
* - (optional) BATCH_SIZE: Number of blocks to fetch in a single batch request (default: 100)
* - (optional) MAX_RETRIES: Maximum number of retries for failed requests (default: 5)
* - (optional) INITIAL_RETRY_DELAY: Initial delay in ms before retrying (default: 1000)
*/

// requires ethers v5
const { ethers } = require('ethers');
const fs = require('fs');
const path = require('path');

/**
 * Fetches multiple blocks in a single batch request with retry logic
 * @param {ethers.providers.JsonRpcProvider} provider - The provider to use
 * @param {number} fromBlock - Starting block number
 * @param {number} batchSize - Number of blocks to fetch
 * @param {object} options - Options including retry parameters
 * @returns {Promise<Array>} - Array of block data
 */
async function fetchBlocksBatch(provider, fromBlock, batchSize, options) {
  const { maxRetries, initialRetryDelay, debugMode } = options;
  let retries = 0;
  let delay = initialRetryDelay;

  while (true) {
    try {
      const requests = [];
      
      for (let i = 0; i < batchSize; i++) {
        const blockNumber = fromBlock + i;
        const blockNumberHex = '0x' + blockNumber.toString(16);
        
        // Create a request for each block
        requests.push({
          jsonrpc: '2.0',
          id: i,
          method: 'eth_getBlockByNumber',
          params: [blockNumberHex, true] // true to include transactions
        });
      }
      
      // Send the batch request
      const results = await provider.send('eth_getBatchRequest', requests);
      
      // Some providers don't support batch requests directly, try an alternative
      if (!results || !Array.isArray(results)) {
        // Fallback to sending individual requests in parallel
        const promises = requests.map(req => 
          provider.send(req.method, req.params)
            .catch(err => {
              throw new Error(`Error fetching block ${parseInt(req.params[0], 16)}: ${err.message}`);
            })
        );
        
        return await Promise.all(promises);
      }
      
      return results.map(result => result.result);
    } catch (error) {
      retries++;
      if (retries > maxRetries) {
        throw new Error(`Maximum retries (${maxRetries}) exceeded: ${error.message}`);
      }
      
      if (debugMode) {
        console.log(`[DEBUG] Attempt ${retries}/${maxRetries} failed for blocks ${fromBlock}-${fromBlock+batchSize-1}: ${error.message}`);
        console.log(`[DEBUG] Retrying in ${delay}ms...`);
      }
      
      // Wait with exponential back-off
      await new Promise(resolve => setTimeout(resolve, delay));
      
      // Exponential back-off: double the delay for next retry (with some randomness)
      delay = delay * 2 * (0.75 + Math.random() * 0.5);
    }
  }
}

/**
 * Fetches blocks with transaction data with retry logic
 * @param {ethers.providers.JsonRpcProvider} provider - The provider to use
 * @param {number} fromBlock - Starting block number
 * @param {number} batchSize - Number of blocks to fetch
 * @param {object} options - Retry options
 * @returns {Promise<Array>} - Array of blocks with transactions
 */
async function fetchBlocksWithRetry(provider, fromBlock, batchSize, options) {
  const { maxRetries, initialRetryDelay, debugMode } = options;
  let retries = 0;
  let delay = initialRetryDelay;

  while (true) {
    try {
      const promises = [];
      for (let i = 0; i < batchSize; i++) {
        promises.push(
          provider.getBlockWithTransactions(fromBlock + i)
        );
      }
      return await Promise.all(promises);
    } catch (error) {
      retries++;
      if (retries > maxRetries) {
        throw new Error(`Maximum retries (${maxRetries}) exceeded: ${error.message}`);
      }
      
      if (debugMode) {
        console.log(`[DEBUG] Attempt ${retries}/${maxRetries} failed for blocks ${fromBlock}-${fromBlock+batchSize-1}: ${error.message}`);
        console.log(`[DEBUG] Retrying in ${delay}ms...`);
      }
      
      // Wait with exponential back-off
      await new Promise(resolve => setTimeout(resolve, delay));
      
      // Exponential back-off: double the delay for next retry (with some randomness)
      delay = delay * 2 * (0.75 + Math.random() * 0.5);
    }
  }
}

async function main() {
  // Get RPC URL from environment variable
  const rpcUrl = process.env.RPC;
  if (!rpcUrl) {
    console.error('Please set the RPC environment variable');
    process.exit(1);
  }

  // Get number of addresses to collect from environment variable
  const nrAddrs = parseInt(process.env.NR_ADDRS || '10');
  if (isNaN(nrAddrs) || nrAddrs <= 0) {
    console.error('NR_ADDRS must be a positive number');
    process.exit(1);
  }

  // Get starting block from environment variable (default: 1)
  const startBlock = parseInt(process.env.START_BLOCK || '1');
  if (isNaN(startBlock) || startBlock <= 0) {
    console.error('START_BLOCK must be a positive number');
    process.exit(1);
  }
  
  // Get batch size from environment variable (default: 100)
  const batchSize = parseInt(process.env.BATCH_SIZE || '100');
  if (isNaN(batchSize) || batchSize <= 0) {
    console.error('BATCH_SIZE must be a positive number');
    process.exit(1);
  }
  
  // Get max retries from environment variable (default: 5)
  const maxRetries = parseInt(process.env.MAX_RETRIES || '5');
  if (isNaN(maxRetries) || maxRetries < 0) {
    console.error('MAX_RETRIES must be a non-negative number');
    process.exit(1);
  }
  
  // Get initial retry delay from environment variable (default: 1000ms)
  const initialRetryDelay = parseInt(process.env.INITIAL_RETRY_DELAY || '1000');
  if (isNaN(initialRetryDelay) || initialRetryDelay < 0) {
    console.error('INITIAL_RETRY_DELAY must be a non-negative number');
    process.exit(1);
  }

  // Check if debug mode is enabled
  const debugMode = process.env.DEBUG === '1';

  // Connect to the provider
  const provider = new ethers.providers.JsonRpcProvider(rpcUrl);
  
  // Get the network information to determine chainId
  const network = await provider.getNetwork();
  const chainId = network.chainId;
  console.log(`Connected to network with chainId: ${chainId}`);
  
  // Set to store unique sender addresses
  const uniqueAddresses = new Set();
  let blockNumber = startBlock;
  
  console.log(`Collecting ${nrAddrs} unique addresses from transactions starting at block ${blockNumber}...`);
  console.log(`Using batch size of ${batchSize} blocks per request`);
  console.log(`Max retries: ${maxRetries}, Initial retry delay: ${initialRetryDelay}ms`);
  
  // Retry configuration
  const retryOptions = {
    maxRetries,
    initialRetryDelay,
    debugMode
  };
  
  try {
    // Use batch requests when supported, otherwise fallback to our custom implementation
    let useBatchMethod = true;
    
    // Try to determine if the provider supports batch requests
    try {
      // Try to send a small batch to test functionality
      await provider.send('eth_getBatchRequest', [
        { jsonrpc: '2.0', id: 1, method: 'eth_blockNumber', params: [] }
      ]);
    } catch (error) {
      if (debugMode) {
        console.log('[DEBUG] Provider does not support eth_getBatchRequest, using custom batch implementation');
      }
      useBatchMethod = false;
    }
    
    while (uniqueAddresses.size < nrAddrs) {
      try {
        let blocks = [];
        
        if (useBatchMethod) {
          // Use the batch request method with retry logic
          const batchResults = await fetchBlocksBatch(provider, blockNumber, batchSize, retryOptions);
          blocks = batchResults.filter(block => block !== null);
        } else {
          // Fallback to fetching blocks in parallel with retry logic
          blocks = await fetchBlocksWithRetry(provider, blockNumber, batchSize, retryOptions);
          blocks = blocks.filter(block => block !== null);
        }
        
        if (blocks.length === 0) {
          console.log(`No blocks found at ${blockNumber}, moving to next batch.`);
          blockNumber += batchSize;
          continue;
        }
        
        for (const block of blocks) {
          if (!block || !block.transactions) {
            blockNumber++;
            continue;
          }
          
          const currentBlockNumber = block.number || parseInt(block.number, 16);
          
          // For debug mode: collect block-specific stats
          if (debugMode) {
            const initialSize = uniqueAddresses.size;
            const blockSenders = new Set();
            
            // Process transactions in the block for debug information
            for (const tx of block.transactions) {
              if (tx.from) {
                const from = typeof tx.from === 'string' ? tx.from.toLowerCase() : tx.from.toLowerCase();
                blockSenders.add(from);
              }
            }
            
            // Add all block senders to our main set
            blockSenders.forEach(addr => uniqueAddresses.add(addr));
            
            // Calculate the change in set size
            const sizeChange = uniqueAddresses.size - initialSize;
            
            console.log(`[DEBUG] Block ${currentBlockNumber}: ${block.transactions.length} txs, ${blockSenders.size} unique senders, set size change: +${sizeChange} (now ${uniqueAddresses.size})`);
          } else {
            // Normal processing without debug info
            for (const tx of block.transactions) {
              if (tx.from) {
                const from = typeof tx.from === 'string' ? tx.from.toLowerCase() : tx.from.toLowerCase();
                uniqueAddresses.add(from);
                
                // Log progress periodically
                if (uniqueAddresses.size % 10 === 0) {
                  console.log(`Collected ${uniqueAddresses.size}/${nrAddrs} addresses (at block ${currentBlockNumber})`);
                }
                
                // Break if we've reached the target
                if (uniqueAddresses.size >= nrAddrs) {
                  break;
                }
              }
            }
          }
          
          // Check if we've reached the target
          if (uniqueAddresses.size >= nrAddrs) {
            console.log(`Reached target of ${nrAddrs} addresses at block ${currentBlockNumber}`);
            break;
          }
          
          blockNumber = currentBlockNumber + 1;
        }
        
        // If we've processed all blocks in the batch but haven't reached our target
        if (uniqueAddresses.size < nrAddrs) {
          blockNumber = Math.max(blockNumber, startBlock + batchSize);
        }
        
      } catch (error) {
        console.error(`Error processing blocks starting at ${blockNumber}:`, error.message);
        // Don't increment block number here - we want to retry the same blocks
        // Just add a small delay before retry
        await new Promise(resolve => setTimeout(resolve, 1000));
      }
    }
    
    // Convert the set to an array for output
    const addressesArray = Array.from(uniqueAddresses);
    
    // Create file name with chainId and number of addresses
    const fileName = `addresses_chain_${chainId}_${addressesArray.length}.txt`;
    
    // Write addresses to file, one per line
    fs.writeFileSync(fileName, addressesArray.join('\n'));
    console.log(`\nAddresses saved to ${fileName}`);
    
    if (debugMode) {
      // Output the collected addresses to console
      console.log('\nCollected addresses:');
      addressesArray.forEach(addr => console.log(addr));
      console.log(`\nTotal: ${addressesArray.length} unique addresses`);
    }
    
  } catch (error) {
    console.error('An error occurred:', error);
    process.exit(1);
  }
}

main().catch(error => {
  console.error('Unhandled error:', error);
  process.exit(1);
});