Java Code to compare the test-cases in xml for Jenkins and to modify the test-cases

CompareXml.Java

******************************
 
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.File;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.io.PrintWriter;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;

import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
import org.xml.sax.InputSource;

public class CompareXml
{
    public static String readFile(String path, Charset encoding) throws IOException
    {
        byte[] encoded = Files.readAllBytes(Paths.get(path));
        return new String(encoded, encoding);
    }

    public static String stripNonValidXMLCharacters(String in)
    {
        StringBuffer out = new StringBuffer(); // Used to hold the output.
        char current; // Used to reference the current character.

        if (in == null || ("".equals(in)))
            return ""; // vacancy test.
        for (int i = 0; i < in.length(); i++) {
            current = in.charAt(i); // NOTE: No IndexOutOfBoundsException caught here; it should not happen.
            if ((current == 0x9) || (current == 0xA) || (current == 0xD) || ((current >= 0x20) && (current <= 0xD7FF))
                || ((current >= 0xE000) && (current <= 0xFFFD)) || ((current >= 0x10000) && (current <= 0x10FFFF)))
                out.append(current);
        }
        return out.toString();
    }


    public static HashMap parse_xml(String xml_file) throws ParserConfigurationException, SAXException, IOException {
        HashMap xml_data = new HashMap();
        System.out.println(xml_file);

        String content = readFile(xml_file, Charset.defaultCharset());
        content = stripNonValidXMLCharacters(content);
        PrintWriter out = new PrintWriter(xml_file);
        out.println(content);
        out.close();
        InputStream inputStream= new FileInputStream(xml_file);
        Reader reader = new InputStreamReader(inputStream,"UTF-8");
        InputSource is = new InputSource(reader);
        is.setEncoding("UTF-8");

        DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
        DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
        Document doc = dBuilder.parse(is);

        //File fXmlFile = new File(xml_file);
        //DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
        //DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
        //Document doc = dBuilder.parse(new InpputSource(new InputStreamReader(fXmlFile, "UTF-8")));
        doc.getDocumentElement().normalize();
        System.out.println("Root element :" + doc.getDocumentElement().getNodeName());
        NodeList nList = doc.getElementsByTagName("method");
        System.out.println("----------------------------");
        for (int temp = 0; temp < nList.getLength(); temp++) {
            Node nNode = nList.item(temp);
            if (nNode.getNodeType() == Node.ELEMENT_NODE) {
                Element eElement = (Element) nNode;
                String[] parts = eElement.getAttribute("signature").split("@");
                xml_data.put(temp, parts[0]);
            }
        }
        return xml_data;
    }

    public static void compare_xmls() throws Exception
    {

            String currentBuild = System.getProperty("CURRENT_BUILD");
            String previousBuild = "";
            int prev_build_test = Integer.parseInt(currentBuild);
            prev_build_test = prev_build_test - 1;
            previousBuild = String.valueOf(prev_build_test);
            System.out.println("Current Build : "+currentBuild);
            System.out.println("Previous Build : "+previousBuild);

            String path = "/var/lib/jenkins/jobs//builds//testng/testng-results.xml";

            String currentPath = path.replace("", currentBuild);
            String prevPath = path.replace("", previousBuild);


            HashMap xml_data = new HashMap();
            HashMap xml_data2 = new HashMap();
            HashMap final_xml_data = new HashMap();
            HashMap diff_xml_data = new HashMap();
            String usrdir = System.getProperty("user.dir");

            xml_data = parse_xml(currentPath);
            xml_data2 = parse_xml(prevPath);

            System.out.println("First Hashmap Size : "+ xml_data.size());
            System.out.println("Second Hashmap Size : "+ xml_data2.size());

            int temp1=0;
            for (int itr = 0; itr < xml_data.size(); itr++) {
                if (!xml_data2.containsValue(xml_data.get(itr))) {
                    final_xml_data.put(temp1, xml_data.get(itr));
                    temp1 = temp1 +1;
                }
            }
            System.out.println("Newly Added TCs Hashmap Size : "+ final_xml_data.size());

            int temp2 = 0;
            for( String s : xml_data2.values())
            {
                if (!xml_data.containsValue(s)) {
                    diff_xml_data.put(temp2, s);
                    temp2 = temp2 + 1;
                }
            }

            System.out.println("Deleted TCs Hashmap Size : " + diff_xml_data.size());

            HashSet final_xml_data_values = new HashSet();
            final_xml_data_values.addAll(final_xml_data.values());

            HashSet diff_xml_data_values = new HashSet();
            diff_xml_data_values.addAll(diff_xml_data.values());

            System.out.println("Newly Added Test-Cases Below : ");
            for (String str : final_xml_data_values){
                System.out.println("NEW-TC : " + str);
            }

            System.out.println("\n\n********************************");
            for (String str : diff_xml_data_values){
                System.out.println("DEL-TC : " + str);
            }

            System.out.println("Deleted Test-Cases Below : ");

    }
        public static void main(String args[]) throws Exception {
                compare_xmls();
    }

}




*******************************************************************************

ModifyXml.java

****************************

import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.Reader;

import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

import java.io.File;
import java.io.FileOutputStream;
import java.io.StringWriter;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;

import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

public class ModifyXml
{
    public static String readFile(String path, Charset encoding) throws IOException
    {
        byte[] encoded = Files.readAllBytes(Paths.get(path));
        return new String(encoded, encoding);
    }

    public static String stripNonValidXMLCharacters(String in)
    {
        StringBuffer out = new StringBuffer(); // Used to hold the output.
        char current; // Used to reference the current character.

        if (in == null || ("".equals(in)))
            return ""; // vacancy test.
        for (int i = 0; i < in.length(); i++) {
            current = in.charAt(i); // NOTE: No IndexOutOfBoundsException caught here; it should not happen.
            if ((current == 0x9) || (current == 0xA) || (current == 0xD) || ((current >= 0x20) && (current <= 0xD7FF))
                || ((current >= 0xE000) && (current <= 0xFFFD)) || ((current >= 0x10000) && (current <= 0x10FFFF)))
                out.append(current);
        }
        return out.toString();
    }

    public static void parse_xml(String xml_file, String... test_names) throws Exception
    {
        String content = readFile(xml_file, Charset.defaultCharset());
        content = stripNonValidXMLCharacters(content);
        PrintWriter out = new PrintWriter(xml_file);
        out.println(content);
        out.close();
        InputStream inputStream = new FileInputStream(xml_file);
        Reader reader = new InputStreamReader(inputStream, "UTF-8");
        InputSource is = new InputSource(reader);
        is.setEncoding("UTF-8");

        DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
        DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
        Document doc = dBuilder.parse(is);

        // File fXmlFile = new File(xml_file);
        // DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
        // DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
        // Document doc = dBuilder.parse(fXmlFile);
        doc.getDocumentElement().normalize();
        System.out.println("Root element :" + doc.getDocumentElement().getNodeName());
        NodeList nList = doc.getElementsByTagName("test-method");
        System.out.println("----------------------------");
        for (int temp = 0; temp < nList.getLength(); temp++) {
            Node nNode = nList.item(temp);
            if (nNode.getNodeType() == Node.ELEMENT_NODE) {
                Element eElement = (Element) nNode;
                String status = eElement.getAttribute("status");
                String name = eElement.getAttribute("name");
                for (String test_name : test_names) {
                    if (name.equalsIgnoreCase(test_name)) {
                        System.out.println("Signature : " + eElement.getAttribute("signature"));
                        System.out.println("Status : " + eElement.getAttribute("status"));
                        if (status.contains("FAIL")) {
                            eElement.setAttribute("status", "PASS");
                        }
                        System.out.println("Signature : " + eElement.getAttribute("signature"));
                        System.out.println("Status : " + eElement.getAttribute("status"));
                    }
                }
            }
        }

        Transformer transformer = TransformerFactory.newInstance().newTransformer();
        transformer.setOutputProperty(OutputKeys.INDENT, "yes");
        StreamResult result = new StreamResult(new StringWriter());
        DOMSource source = new DOMSource(doc);
        transformer.transform(source, new StreamResult(new FileOutputStream(xml_file)));

        System.out.println("File updated!");
    }

    public static void modify_xml() throws Exception
    {
        String currentBuild = System.getProperty("CURRENT_BUILD");
        String test_names = System.getProperty("TEST_NAMES");
        String path = "/var/lib/jenkins/jobs//builds//testng/testng-results.xml";
        String currentPath = path.replace("", currentBuild);

        parse_xml(currentPath, test_names.split(","));

    }

    public static void main(String args[]) throws Exception
    {
        modify_xml();
    }

}

Jenkins as the Test Reporting Framework

Very few and matured companies are able to invest on the development of the Test Reporting Frameworks. At Komli we realised the benefits & value it offers.Instead of buying some commercial tools we went with the approach of extending Jenkins for the same. This talk is about our experience regarding the same.
Test Reporting Framework are the ultimate utilities reached by very few companies in the world likes of Google & Facebook. It require significant investment in terms money and resources. To optimize on both we tried the approach of extending Jenkins for the same purpose.
We like to share our experiences with respect to this journey. We will talk about different challenges we faced in doing so and how we solved them.
We strongly believe that other startup can use our experience and get benefit from this immensely.

WHY TEST REPORTING FRAMEWORK?

1. To help with informed decision, whether to go ahead with production deployment of the build
2. To analyze whether sufficient testing being done

• What are code changes?
• What are the corresponding automation changes w.r.t test-cases added/deleted/modified
• What is the test automation run trend w.r.t trend on Release basis

REPORTING FRAMEWORK REQUIREMENTS

• Web based Tool
• Cost Effective
• Easy to maintain
• Integration with Test Case Execution Frameworks

SOME OF PAID TEST REPORTING FRAMEWORKS AVAILABLE

• TestRail
• Xqual
• Test Collab

SOLUTION TO THE ABOVE PROBLEMS :- JENKINS

• Continuous Integration server that allows to run multiple automation suites at the same time.
• Supports SVN, Git and about any other SCM tools you can think of via plugins

WHY JENKINS

• Web based
• No additional cost, we are already dependent on it as Test-Case Execution Framework
• Well supported
• Highly customizable

JENKINS ON THE RELEASE BASIS

COMMON JENKINS TERMS

• Project
• Builds

NEW TERMS INTRODUCED

• Benchmark Project
• Benchmark Builds 

BENCHMARK PROJECT

• The project in Jenkins where we maintain the automation results on the basis of releases done to production.

BENCHMARK BUILD

• Build which has best results
• The one whose final status we want to maintain in the benchmark project

WHAT IS ALREADY AVAILABLE IN JENKINS

• Show automation run trend
• Integration with SCM tool show code diffs

CHALLENGES

1. Integration : Want everything in the single screen (single project in Jenkins)
2. Benchmark Build : We don't know priori that given build is the best build to run

• Don't want to waste resources and time to execute automation run again once we have decided on the build to deploy on production.
• Intermittent Failures : Want to maintain their status as Passed for a given release
• If we have taken the approach of copying results from one project to other project (benchmark project).

OUR APPROACH

• Git Plugin for Code Diff
• Custom Java Script for Test-Cases diff
• Custom Java Script for overriding intermittent failure status
• CopyArtifact Plugin to migrate results
• Custom Groovy Script for re-generating the TestNG result trends

INTERMITTENT FAILURES

 

PARAMETERS TO COPY BENCHMARK BUILD

Slideshare Link for Presentation at RootConf