& Continuous Learning

You can't ignore the fact that web servers are multithreaded. We can hide as much as we want, but sooner or later you'll find yourself in the situation where your application works fine during development and testing; but once it hits production you start hearing about "funny" things happening. While there are plenty of tools that can be used to simulate multiple users, they aren't always the easiest to run locally and they seem to take to much time to modify while hot on the trail of a multithreading bug. Here I'll discuss the approach I took when I was recently faced with this situation.

It was the end of the quarter and teachers were complaining that the test scoring application was returning incorrect results. The application had passed all development tests, passed through qa with flying colors, and has been deployed without any issue for a little while now. To make matters worse, all of the teachers were under lots of pressure to get all of their tests scored before the end of the quarter. When I heard that, a little light went off in my head. With a little digging, we were able to determine that the system had been serving more concurrent users than planned for.

The first thing I try to do when faced with a bug report is find a way to consistently reproduce it. This particular issue is a little harder since it only seems to be happening with multiple users. Luckily, Java makes writing multithreaded code very easy. Since most of the complaints seem to be related to test scoring, I start focusing my attention on that. The plan of attack is to:

1. Get a list of test ID's.
2. Score each test and save it's result.
3. Create a separate thread for each ID.
4. Each thread will score the test again, and compare it to the score from step 2. If it doesn't match, increment a failure count.
5. Perform step 4 a couple of times.
6. Once all of the threads exit, print out the number of failures.

The plan of attack would then be to run this until either a failure occurs or until I feel fairly certain that there isn't a problem with the scoring service.

Here is the the initial version of the test:

package com.sourceallies.multithread;

import java.util.ArrayList;
import java.util.List;

public class ScoringServiceTest {

 private int runCount = 0;
 private int scoreFailureCount = 0;
 private List skipFailureList = new ArrayList();
 private ScoringService scoringService = new ScoringService();


 private synchronized void incrementRunCount() {
  runCount++;
 }
 private synchronized void incrementScoreFailureCount() {
  scoreFailureCount++;
 }

 private int getRunCount() {
  return runCount;
 }
 private int getScoreFailureCount() {
  return scoreFailureCount;
 }

 private boolean compareScores(ScoringResult a, ScoringResult b) {
  return a.getNumerator().equals(b.getNumerator()) &&
  a.getDenominator().equals(b.getDenominator());
 }


 private void runTest(List testIdList, int iterationCount) {
  long start = System.currentTimeMillis();

  List threadList = new ArrayList();
  String testId = "";
  for(int i = 0; i < testIdList.size(); i++) {
   try {
   testId = testIdList.get(i);
   ScoringResult score = scoringService.scoreTest(testId);
   Runnable testRunner = new TestRunner(iterationCount, testId, score);
   Thread thread = new Thread(testRunner, testId);
   threadList.add(thread);

   }
   catch(Exception exc) {
    System.out.println("Error with ID: " + testId);
    exc.printStackTrace();
   }
  }

  System.out.println("Starting threads");
  for(Thread t : threadList) {
   t.start();
  }

  System.out.println("Waiting for threads to finish");
  for(Thread t : threadList) {
   try {
    t.join();
   }
   catch(InterruptedException exc) {
    System.out.println("Interrupted while waiting for " + t.getName());
   }
  }

  System.out.println("Number of IDs: " + testIdList.size());
  System.out.println("Iterations: " + iterationCount);
  System.out.println("Total: " + getRunCount());
  System.out.println("Elapsed Time: " + (System.currentTimeMillis() - start));
  System.out.println("# Score Failures: " + getScoreFailureCount());
 }

 private class TestRunner implements Runnable {
  private String testId;
  private ScoringResult expectedScore;
  private int iterations;

  TestRunner(int iters, String tId, ScoringResult score) {
   testId = tId;
   expectedScore = score;
   iterations = iters;
  }

  @Override
  public void run() {

   for(int i = 0; i < iterations; i++) {
    ScoringServiceTest.this.incrementRunCount();
    ScoringResult scoringResult = ScoringServiceTest.this.scoringService.scoreTest(testId);
    if(!compareScores(scoringResult, expectedScore)) {
     ScoringServiceTest.this.incrementScoreFailureCount();
    }
   }
  }
 }

 public static void main(String[] args) {
  ScoringServiceTest test = new ScoringServiceTest();
  List idList = new ArrayList();

  idList.add("Math1");
  idList.add("Science1");
  idList.add("Math2");
  idList.add("Science2");
  idList.add("English1");
  test.runTest(idList, 500);
 }
}

While this did the job at first, it's usefulness was very short lived. Before long, I was wanting to change the test logic and didn't want the threading logic getting in the way. The approach I took was to extract the threading code out into a test harness and then create an interface for the testing logic by using the Command pattern. This resulted in the following code, which I have been able to reuse across multiple test scenarios.

The test harness class:

package com.sourceallies.multithread;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;

public class MultithreadedTest {

 private int runCount = 0;
 private int failureCount = 0;


 private synchronized void incrementRunCount() {
  runCount++;
 }
 private synchronized void incrementFailureCount() {
  failureCount++;
 }
 public int getRunCount() {
  return runCount;
 }
 public int getFailureCount() {
  return failureCount;
 }


 public void runTest(Collection commands, int iterationCount) {
  long start = System.currentTimeMillis();

  List threadList = new ArrayList();
  System.out.println("Loading expected results");

  for(TestCommand command : commands) {
   Object result = command.doTest();
   TestRunner testRunner = new TestRunner(iterationCount, command, result);
   Thread thread = new Thread(testRunner);
   threadList.add(thread);
  }

  System.out.println("Starting threads");
  for(Thread t : threadList) {
   t.start();
  }

  System.out.println("Waiting for threads to finish");
  for(Thread t : threadList) {
   try {
    t.join();
   }
   catch(InterruptedException exc) {
    System.out.println("Interrupted while waiting for " + t.getName());
   }
  }

  System.out.println("Number of Tests: " + commands.size());
  System.out.println("Iterations: " + iterationCount);
  System.out.println("Total: " + getRunCount());
  System.out.println("Elapsed Time: " + (System.currentTimeMillis() - start));
  System.out.println("# Failures: " + getFailureCount());
 }


 private class TestRunner implements Runnable {
  private int iterations;
  private TestCommand command;
  private Object expectedResult;

  TestRunner(int iters, TestCommand command, Object expectedResult) {
   this.command = command;
   iterations = iters;
   this.expectedResult = expectedResult;
  }

  @Override
  public void run() {

   for(int i = 0; i < iterations; i++) {
    MultithreadedTest.this.incrementRunCount();
    try {
     Object result = command.doTest();
     if(!command.compareResults(result, expectedResult)) {
      MultithreadedTest.this.incrementFailureCount();
     }
    }
    catch(Exception exc) {
     exc.printStackTrace();
     MultithreadedTest.this.incrementFailureCount();
    }
   }
  }
 }


}

The test command:

package com.sourceallies.multithread;

public interface TestCommand {
 public Object doTest();
 public boolean compareResults(Object a, Object b);
}

and finally, the test command for my scenario:

package com.sourceallies.multithread;

import java.util.ArrayList;
import java.util.List;

public class ScoringServiceTestCommand implements TestCommand {
 private String testId;
 private ScoringService scoringService;

 public ScoringServiceTestCommand(ScoringService service, String id) {
  scoringService = service;
  testId = id;
 }

 public Object doTest() {
  return scoringService.scoreTest(testId);
 }


 public boolean compareResults(Object expectedResult, Object result) {
  ScoringResult sr1 = (ScoringResult)expectedResult;
  ScoringResult sr2 = (ScoringResult)result;

  boolean match = sr1.getNumerator().equals(sr2.getNumerator()) &&
    sr1.getDenominator().equals(sr2.getDenominator());

  return match;
 }

 public static void main(String[] args) {
  List idList = new ArrayList();
  idList.add("Math1");
  idList.add("Science1");
  idList.add("Math2");
  idList.add("Science2");
  idList.add("English1");

  ScoringService scoringService = new ScoringService();

  List commandList = new ArrayList();
  for(int i = 0; i < idList.size(); i++) {
   commandList.add(new ScoringServiceTestCommand(scoringService, idList.get(i)));
  }
  MultithreadedTest mtest = new MultithreadedTest();
  mtest.runTest(commandList, 500);
 }
}