flaky-test-finder.rb

require 'json'
require 'net/http'
require 'optparse'
require 'set'
require 'uri'
require 'octokit'

options = {}
OptionParser.new do |opt|
  opt.on('-s', '--start BUILD_NUMBER', 'Require start') { |o| options[:start] = o }
  opt.on('-e', '--end BUILD_NUMBER', 'Require end') { |o| options[:end] = o }
end.parse!

if options[:start] && options[:end]
  builds =(options[:start]..options[:end]).to_a
else
  jobs_url = "https://build.ci.opensearch.org/job/gradle-check/api/json?tree=builds[number,url]"
  builds = JSON.parse(Net::HTTP.get_response(URI.parse(jobs_url)).body)['builds'].map { |build| build['number'] }
end

puts "Will crawl #{builds.size} builds ..."
all_failed_tests = []
builds.each do |job_number|
  base_job_uri = "https://build.ci.opensearch.org/job/gradle-check/#{job_number}"
  result = JSON.parse(Net::HTTP.get_response(URI.parse(base_job_uri + '/api/json')).body)['result']

  # UNSTABLE means the build succeeded but at least one test needed to be
  # retried. Many gradle-check runs fail when running against a PR because the
  # newly introduced code has a problem. The developer then iterates on the PR
  # until all problems are resolved. To filter out this noise we only consider
  # UNSTABLE builds, and ignore failures. It is possible for gradle-check
  # builds run against merged code to fail due to flaky tests and they would be
  # missed here. However, I think that is a small minority of the cases and
  # this approach should do pretty well at identifying flaky tests.
  if result == 'UNSTABLE'
    uri = URI.parse(base_job_uri + '/testReport/api/json?tree=suites[cases[status,className,name]]')
    json = JSON.parse(Net::HTTP.get_response(uri).body)
    # 'FAILED' means the test failed, just like the previous run.
    # 'REGRESSION' means the test failed, but previously passed.
    # See https://javadoc.jenkins.io/plugin/junit/hudson/tasks/junit/CaseResult.Status.html
    failed_cases = json['suites'].map do |s|
      s['cases'].select do |c|
        c['status'] == 'REGRESSION' || c['status'] == 'FAILED'
      end
    end.flatten
    failed_tests = failed_cases.map { |c| {'name' => "#{c['className']}.#{c['name']}", 'build' => job_number}}
    all_failed_tests.push(failed_tests)
  end
end

puts '------------------'

github_api_token = ENV['GITHUB_API_TOKEN']
github = Octokit::Client.new(access_token: github_api_token) if github_api_token

count = {}
all_failed_tests.flatten.each do |test|
  unless count.include?(test['name'])
    issue = github.search_issues("is:issue in:title repo:opensearch-project/OpenSearch \"#{test['name']}\"").items.first
    issue ||= github.search_issues("is:issue in:title repo:opensearch-project/OpenSearch \"#{test['name'].split('.').last}\"").items.first
    count[test['name']] = {'count' => 0, 'builds' => [], 'issue' => issue}
  end
  count[test['name']]['count'] += 1
  count[test['name']]['builds'].push(test['build'])
end

sorted_count = count.to_a.sort {|a,b| b[1]['count'] <=> a[1]['count'] }

sorted_count.each do |name, data| 
  puts "#{data['count']} #{name} (#{data['builds'].join(',')}) \##{data['issue']&.number}"
end