Retry Logic in TestNG to re-run failed test-cases | TestNG | Jenkins

1| Assume you have a class file with 10 Methods [@Test]
2| Lets say one got failed due to Network issue
3| The class file, RetryAnalyzer will make the failed test-case to re-run with desired count; But, it display errors for all the test failures during re-run.
4| You are ought to add a Listener class, RetryTestListener; it will print the failure only once.
5| If the test got passed in the final count, failure exception won't be thrown in the console.


Test Class

@BeforeTest
public void setUp()
 {
 driver = new FirefoxDriver();
 baseUrl = "https://www.google.co.in";
 driver.manage().timeouts().implicitlyWait(30, TimeUnit.SECONDS);
 }

@Test(retryAnalyzer=RetryAnalyzer.class)
public void test01() throws Exception
 {
 driver.get(baseUrl + "/");
 String save = driver.findElement(By.id("als")).getText();
 System.out.println(save);
        // Assert.fail();
 Assert.assertEquals(save, "qwerty");
 }

**************************************************************

RetryAnalyzer.java

Create a Java class, 'RetryAnalyzer' inside the same package and paste the following snippet with minor edits.

import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;
import org.testng.IRetryAnalyzer;
import org.testng.ITestResult;

public class RetryAnalyzer implements IRetryAnalyzer  {
private int count = 0;
private int maxCount = 4; // set your count to re-run test
protected Logger log;
private static Logger testbaseLog;

static {
    PropertyConfigurator.configure("test-config/log4j.properties");
    testbaseLog = Logger.getLogger("TestclassName");
}

public RetryAnalyzer()
{
    testbaseLog.trace( " ModeledRetryAnalyzer constructor " + this.getClass().getName() );
    log = Logger.getLogger("TestclassName");
}

@Override
public boolean retry(ITestResult result) {
    testbaseLog.trace("running retry logic for  '"
            + result.getName()
            + "' on class " + this.getClass().getName() );
        if(count < maxCount) {                    
                count++;                                   
                return true;
        }
        return false;
}
}

******************************************************

RetryTestListener.java

import org.testng.ITestContext;
import org.testng.ITestNGMethod;
import org.testng.ITestResult;
import org.testng.TestListenerAdapter;

import testng.TestCase;
import java.util.HashSet;
import java.util.Set;

/**
 * Listens for failed tests that need to be rerun.
 */
public class RetryTestListener extends TestListenerAdapter  {
    private static final  Logger logger = Logger.getLogger(RetryTestListener.class);
    private static int count = 1;
    private static final int maxCount = 7;

    @Override
    public void onTestFailure(ITestResult tr) {  
        tr.setAttribute("retry.count", count);
        tr.setAttribute("retry.maxCount", maxCount);
        boolean cond = false;
        if(count < maxCount) {
            count++;
            try  {
                if(TestCase.driver == null)  {
                    tr.setStatus(ITestResult.SKIP);
                    return;
                }
              
                 cond = true;
                if(cond)  {
                    tr.setAttribute("retry", true);
                }
            } catch(Exception e)  {
                logger.error("COULD NOT RETRY TESTCASE: "+e.getMessage());
            }

        } else  {
            logger.error("Number of retries expired.");
            tr.setStatus(ITestResult.FAILURE);
            // reset count
            count = 1;
        }
        super.onTestFailure(tr);
    }

    @Override
    public void onTestSuccess(ITestResult tr) {
        super.onTestSuccess(tr);
        count = 1;
    }
   
   
    @Override
    public void onFinish(ITestContext context) {
        for (int i = 0; i < context.getAllTestMethods().length; i++) {
            if (context.getAllTestMethods()[i].getCurrentInvocationCount() == 2) {
                if (context.getFailedTests().getResults(context.getAllTestMethods()[i]).size() == 2
                        || context.getPassedTests().getResults(context.getAllTestMethods()[i]).size() == 1) {
                    context.getFailedTests().removeResult(context.getAllTestMethods()[i]);
                }
            }
        }
       
    }
   
    private Set findDuplicates(Set listContainingDuplicates) {
        Set toRemove = new HashSet();
        Set testSet = new HashSet();
       
        for(ITestResult test : listContainingDuplicates)  {
            if (!testSet.add(test.getMethod())) {
                toRemove.add(test.getMethod());
            }   
        }
        return toRemove;
       
    }
}

Custom Groovy Script to display your details on Jenkins Build

pattern = ~/^.*\b("NEW-TC|NEW-TC)\b.*$/
def map = [:]
manager.build.logFile.eachLine { line ->
matcher = pattern.matcher(line)
if(matcher.matches()) {
       ownClass = matcher.group(0)
        sunClass = matcher.group(1)
        map[ownClass] = sunClass
}
}
if(map.size() > 0) {
   summary = manager.createSummary("notepad.png")
   summary.appendText("NEWLY ADDED TEST CASES !!!!", false, true, false, "black")
   map.each{ k, v -> summary.appendText("${k}", false) }
summary.appendText("", false)
}

pattern2 = ~/^.*\b("DEL-TC|DEL-TC)\b.*$/
def map2 = [:]
manager.build.logFile.eachLine { line ->
matcher2 = pattern2.matcher(line)
if(matcher2.matches()) {
       ownClass = matcher2.group(0)
        sunClass = matcher2.group(1)
        map2[ownClass] = sunClass
}
}
if(map2.size() > 0) {
   summary2 = manager.createSummary("notepad.png")
   summary2.appendText("DELETED TEST CASES !!!!", false, true, false, "black")
   map2.each{ k, v -> summary2.appendText("${k}", false) }
summary2.appendText("", false)
}

Java Code to compare the test-cases in xml for Jenkins and to modify the test-cases

CompareXml.Java

******************************
 
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.File;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.io.PrintWriter;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;

import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
import org.xml.sax.InputSource;

public class CompareXml
{
    public static String readFile(String path, Charset encoding) throws IOException
    {
        byte[] encoded = Files.readAllBytes(Paths.get(path));
        return new String(encoded, encoding);
    }

    public static String stripNonValidXMLCharacters(String in)
    {
        StringBuffer out = new StringBuffer(); // Used to hold the output.
        char current; // Used to reference the current character.

        if (in == null || ("".equals(in)))
            return ""; // vacancy test.
        for (int i = 0; i < in.length(); i++) {
            current = in.charAt(i); // NOTE: No IndexOutOfBoundsException caught here; it should not happen.
            if ((current == 0x9) || (current == 0xA) || (current == 0xD) || ((current >= 0x20) && (current <= 0xD7FF))
                || ((current >= 0xE000) && (current <= 0xFFFD)) || ((current >= 0x10000) && (current <= 0x10FFFF)))
                out.append(current);
        }
        return out.toString();
    }


    public static HashMap parse_xml(String xml_file) throws ParserConfigurationException, SAXException, IOException {
        HashMap xml_data = new HashMap();
        System.out.println(xml_file);

        String content = readFile(xml_file, Charset.defaultCharset());
        content = stripNonValidXMLCharacters(content);
        PrintWriter out = new PrintWriter(xml_file);
        out.println(content);
        out.close();
        InputStream inputStream= new FileInputStream(xml_file);
        Reader reader = new InputStreamReader(inputStream,"UTF-8");
        InputSource is = new InputSource(reader);
        is.setEncoding("UTF-8");

        DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
        DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
        Document doc = dBuilder.parse(is);

        //File fXmlFile = new File(xml_file);
        //DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
        //DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
        //Document doc = dBuilder.parse(new InpputSource(new InputStreamReader(fXmlFile, "UTF-8")));
        doc.getDocumentElement().normalize();
        System.out.println("Root element :" + doc.getDocumentElement().getNodeName());
        NodeList nList = doc.getElementsByTagName("method");
        System.out.println("----------------------------");
        for (int temp = 0; temp < nList.getLength(); temp++) {
            Node nNode = nList.item(temp);
            if (nNode.getNodeType() == Node.ELEMENT_NODE) {
                Element eElement = (Element) nNode;
                String[] parts = eElement.getAttribute("signature").split("@");
                xml_data.put(temp, parts[0]);
            }
        }
        return xml_data;
    }

    public static void compare_xmls() throws Exception
    {

            String currentBuild = System.getProperty("CURRENT_BUILD");
            String previousBuild = "";
            int prev_build_test = Integer.parseInt(currentBuild);
            prev_build_test = prev_build_test - 1;
            previousBuild = String.valueOf(prev_build_test);
            System.out.println("Current Build : "+currentBuild);
            System.out.println("Previous Build : "+previousBuild);

            String path = "/var/lib/jenkins/jobs//builds//testng/testng-results.xml";

            String currentPath = path.replace("", currentBuild);
            String prevPath = path.replace("", previousBuild);


            HashMap xml_data = new HashMap();
            HashMap xml_data2 = new HashMap();
            HashMap final_xml_data = new HashMap();
            HashMap diff_xml_data = new HashMap();
            String usrdir = System.getProperty("user.dir");

            xml_data = parse_xml(currentPath);
            xml_data2 = parse_xml(prevPath);

            System.out.println("First Hashmap Size : "+ xml_data.size());
            System.out.println("Second Hashmap Size : "+ xml_data2.size());

            int temp1=0;
            for (int itr = 0; itr < xml_data.size(); itr++) {
                if (!xml_data2.containsValue(xml_data.get(itr))) {
                    final_xml_data.put(temp1, xml_data.get(itr));
                    temp1 = temp1 +1;
                }
            }
            System.out.println("Newly Added TCs Hashmap Size : "+ final_xml_data.size());

            int temp2 = 0;
            for( String s : xml_data2.values())
            {
                if (!xml_data.containsValue(s)) {
                    diff_xml_data.put(temp2, s);
                    temp2 = temp2 + 1;
                }
            }

            System.out.println("Deleted TCs Hashmap Size : " + diff_xml_data.size());

            HashSet final_xml_data_values = new HashSet();
            final_xml_data_values.addAll(final_xml_data.values());

            HashSet diff_xml_data_values = new HashSet();
            diff_xml_data_values.addAll(diff_xml_data.values());

            System.out.println("Newly Added Test-Cases Below : ");
            for (String str : final_xml_data_values){
                System.out.println("NEW-TC : " + str);
            }

            System.out.println("\n\n********************************");
            for (String str : diff_xml_data_values){
                System.out.println("DEL-TC : " + str);
            }

            System.out.println("Deleted Test-Cases Below : ");

    }
        public static void main(String args[]) throws Exception {
                compare_xmls();
    }

}




*******************************************************************************

ModifyXml.java

****************************

import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.Reader;

import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

import java.io.File;
import java.io.FileOutputStream;
import java.io.StringWriter;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;

import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

public class ModifyXml
{
    public static String readFile(String path, Charset encoding) throws IOException
    {
        byte[] encoded = Files.readAllBytes(Paths.get(path));
        return new String(encoded, encoding);
    }

    public static String stripNonValidXMLCharacters(String in)
    {
        StringBuffer out = new StringBuffer(); // Used to hold the output.
        char current; // Used to reference the current character.

        if (in == null || ("".equals(in)))
            return ""; // vacancy test.
        for (int i = 0; i < in.length(); i++) {
            current = in.charAt(i); // NOTE: No IndexOutOfBoundsException caught here; it should not happen.
            if ((current == 0x9) || (current == 0xA) || (current == 0xD) || ((current >= 0x20) && (current <= 0xD7FF))
                || ((current >= 0xE000) && (current <= 0xFFFD)) || ((current >= 0x10000) && (current <= 0x10FFFF)))
                out.append(current);
        }
        return out.toString();
    }

    public static void parse_xml(String xml_file, String... test_names) throws Exception
    {
        String content = readFile(xml_file, Charset.defaultCharset());
        content = stripNonValidXMLCharacters(content);
        PrintWriter out = new PrintWriter(xml_file);
        out.println(content);
        out.close();
        InputStream inputStream = new FileInputStream(xml_file);
        Reader reader = new InputStreamReader(inputStream, "UTF-8");
        InputSource is = new InputSource(reader);
        is.setEncoding("UTF-8");

        DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
        DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
        Document doc = dBuilder.parse(is);

        // File fXmlFile = new File(xml_file);
        // DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
        // DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
        // Document doc = dBuilder.parse(fXmlFile);
        doc.getDocumentElement().normalize();
        System.out.println("Root element :" + doc.getDocumentElement().getNodeName());
        NodeList nList = doc.getElementsByTagName("test-method");
        System.out.println("----------------------------");
        for (int temp = 0; temp < nList.getLength(); temp++) {
            Node nNode = nList.item(temp);
            if (nNode.getNodeType() == Node.ELEMENT_NODE) {
                Element eElement = (Element) nNode;
                String status = eElement.getAttribute("status");
                String name = eElement.getAttribute("name");
                for (String test_name : test_names) {
                    if (name.equalsIgnoreCase(test_name)) {
                        System.out.println("Signature : " + eElement.getAttribute("signature"));
                        System.out.println("Status : " + eElement.getAttribute("status"));
                        if (status.contains("FAIL")) {
                            eElement.setAttribute("status", "PASS");
                        }
                        System.out.println("Signature : " + eElement.getAttribute("signature"));
                        System.out.println("Status : " + eElement.getAttribute("status"));
                    }
                }
            }
        }

        Transformer transformer = TransformerFactory.newInstance().newTransformer();
        transformer.setOutputProperty(OutputKeys.INDENT, "yes");
        StreamResult result = new StreamResult(new StringWriter());
        DOMSource source = new DOMSource(doc);
        transformer.transform(source, new StreamResult(new FileOutputStream(xml_file)));

        System.out.println("File updated!");
    }

    public static void modify_xml() throws Exception
    {
        String currentBuild = System.getProperty("CURRENT_BUILD");
        String test_names = System.getProperty("TEST_NAMES");
        String path = "/var/lib/jenkins/jobs//builds//testng/testng-results.xml";
        String currentPath = path.replace("", currentBuild);

        parse_xml(currentPath, test_names.split(","));

    }

    public static void main(String args[]) throws Exception
    {
        modify_xml();
    }

}