View Javadoc
1   /**
2    * Logback: the reliable, generic, fast and flexible logging framework.
3    * Copyright (C) 1999-2015, QOS.ch. All rights reserved.
4    *
5    * This program and the accompanying materials are dual-licensed under
6    * either the terms of the Eclipse Public License v1.0 as published by
7    * the Eclipse Foundation
8    *
9    *   or (per the licensee's choosing)
10   *
11   * under the terms of the GNU Lesser General Public License version 2.1
12   * as published by the Free Software Foundation.
13   */
14  package ch.qos.logback.core.rolling;
15  
16  import static ch.qos.logback.core.CoreConstants.DAILY_DATE_PATTERN;
17  import static org.junit.Assert.assertEquals;
18  import static org.junit.Assert.assertTrue;
19  
20  import java.io.File;
21  import java.io.FileFilter;
22  import java.util.ArrayList;
23  import java.util.Calendar;
24  import java.util.Collections;
25  import java.util.Comparator;
26  import java.util.Date;
27  import java.util.HashSet;
28  import java.util.List;
29  import java.util.Set;
30  import java.util.regex.Matcher;
31  import java.util.regex.Pattern;
32  
33  import org.joda.time.DateTimeZone;
34  import org.joda.time.Days;
35  import org.joda.time.LocalDate;
36  import org.junit.Before;
37  import org.junit.Ignore;
38  import org.junit.Test;
39  
40  import ch.qos.logback.core.CoreConstants;
41  import ch.qos.logback.core.pattern.SpacePadder;
42  import ch.qos.logback.core.rolling.helper.RollingCalendar;
43  import ch.qos.logback.core.rolling.testUtil.ScaffoldingForRollingTests;
44  import ch.qos.logback.core.testUtil.StatusChecker;
45  import ch.qos.logback.core.util.FileSize;
46  import ch.qos.logback.core.util.FixedRateInvocationGate;
47  import ch.qos.logback.core.util.StatusPrinter;
48  
49  public class TimeBasedRollingWithArchiveRemoval_Test extends ScaffoldingForRollingTests {
50      String MONTHLY_DATE_PATTERN = "yyyy-MM";
51      String MONTHLY_CRONOLOG_DATE_PATTERN = "yyyy/MM";
52      final String DAILY_CRONOLOG_DATE_PATTERN = "yyyy/MM/dd";
53  
54      RollingFileAppender<Object> rfa = new RollingFileAppender<Object>();
55      TimeBasedRollingPolicy<Object> tbrp = new TimeBasedRollingPolicy<Object>();
56  
57      // by default tbfnatp is an instance of DefaultTimeBasedFileNamingAndTriggeringPolicy
58      TimeBasedFileNamingAndTriggeringPolicy<Object> tbfnatp = new DefaultTimeBasedFileNamingAndTriggeringPolicy<Object>();
59  
60      StatusChecker checker = new StatusChecker(context);
61  
62      static long MILLIS_IN_MINUTE = 60 * 1000;
63      static long MILLIS_IN_HOUR = 60 * MILLIS_IN_MINUTE;
64      static long MILLIS_IN_DAY = 24 * MILLIS_IN_HOUR;
65      static long MILLIS_IN_MONTH = (long) ((365.242199 / 12) * MILLIS_IN_DAY);
66      static int MONTHS_IN_YEAR = 12;
67  
68      // Wed Mar 23 23:07:05 CET 2016
69      static final long WED_2016_03_23_T_230705_CET = 1458770825333L;
70      static final long THU_2016_03_17_T_230330_CET = 1458252210975L;
71  
72      int slashCount = 0;
73      int ticksPerPeriod = 216;
74  
75      ConfigParameters cp; // initialized in setup
76      FixedRateInvocationGate fixedRateInvocationGate = new FixedRateInvocationGate(ticksPerPeriod / 2);
77  
78      @Before
79      public void setUp() {
80          super.setUp();
81          this.cp = new ConfigParameters(currentTime);
82      }
83  
84      private int computeSlashCount(String datePattern) {
85          if (datePattern == null)
86              return 0;
87          else {
88              int count = 0;
89              for (int i = 0; i < datePattern.length(); i++) {
90                  char c = datePattern.charAt(i);
91                  if (c == '/')
92                      count++;
93              }
94              return count;
95          }
96      }
97  
98      // test that the number of files at the end of the test is same as the expected number taking into account end dates
99      // near the beginning of a new year. This test has been run in a loop with start date varying over a two years
100     // with success.
101     @Test
102     public void monthlyRolloverOverManyPeriods() {
103         this.slashCount = computeSlashCount(MONTHLY_CRONOLOG_DATE_PATTERN);
104         int maxHistory = 2;
105         int simulatedNumberOfPeriods = 30;
106         String fileNamePattern = randomOutputDir + "/%d{" + MONTHLY_CRONOLOG_DATE_PATTERN + "}/clean.txt.zip";
107 
108         cp.maxHistory(maxHistory).fileNamePattern(fileNamePattern).simulatedNumberOfPeriods(simulatedNumberOfPeriods).periodDurationInMillis(MILLIS_IN_MONTH);
109 
110         long startTime = currentTime;
111         long endTime = logOverMultiplePeriods(cp);
112         System.out.println("randomOutputDir:" + randomOutputDir);
113         System.out.println("start:" + startTime + ", end=" + endTime);
114         int differenceInMonths = RollingCalendar.diffInMonths(startTime, endTime);
115         System.out.println("differenceInMonths:" + differenceInMonths);
116         Calendar startTimeAsCalendar = Calendar.getInstance();
117         startTimeAsCalendar.setTimeInMillis(startTime);
118         int indexOfStartPeriod = startTimeAsCalendar.get(Calendar.MONTH);
119         boolean withExtraFolder = extraFolder(differenceInMonths, MONTHS_IN_YEAR, indexOfStartPeriod, maxHistory);
120 
121         checkFileCount(expectedCountWithFolders(maxHistory, withExtraFolder));
122     }
123 
124     long generateDailyRollover(ConfigParameters cp) {
125         this.slashCount = computeSlashCount(DAILY_DATE_PATTERN);
126         cp.fileNamePattern(randomOutputDir + "clean-%d{" + DAILY_DATE_PATTERN + "}.txt");
127         return logOverMultiplePeriods(cp);
128     }
129 
130     long generateDailyRolloverAndCheckFileCount(ConfigParameters cp) {
131         long millisAtEnd = generateDailyRollover(cp);
132         int periodBarriersCrossed = computeCrossedDayBarriers(currentTime, millisAtEnd);
133         System.out.println("**** periodBarriersCrossed=" + periodBarriersCrossed);
134         checkFileCount(expectedCountWithoutFoldersWithInactivity(cp.maxHistory, periodBarriersCrossed, cp.startInactivity + cp.numInactivityPeriods));
135         return millisAtEnd;
136     }
137 
138     @Test
139     public void checkCrossedPeriodsWithDSTBarrier() {
140         long SAT_2016_03_26_T_230705_CET = WED_2016_03_23_T_230705_CET + 3 * CoreConstants.MILLIS_IN_ONE_DAY;
141         System.out.println("SAT_2016_03_26_T_230705_CET " + new Date(SAT_2016_03_26_T_230705_CET));
142         long MON_2016_03_28_T_000705_CET = SAT_2016_03_26_T_230705_CET + CoreConstants.MILLIS_IN_ONE_DAY;
143         System.out.println("MON_2016_03_28_T_000705_CET " + new Date(MON_2016_03_28_T_000705_CET));
144 
145         int result = computeCrossedDayBarriers(SAT_2016_03_26_T_230705_CET, MON_2016_03_28_T_000705_CET, "CET");
146         assertEquals(2, result);
147     }
148 
149     private int computeCrossedDayBarriers(long currentTime, long millisAtEnd) {
150         return computeCrossedDayBarriers(currentTime, millisAtEnd, null);
151     }
152 
153     private int computeCrossedDayBarriers(long currentTime, long millisAtEnd, String timeZoneID) {
154         DateTimeZone dateTimeZone = DateTimeZone.getDefault();
155         if (timeZoneID != null) {
156             dateTimeZone = DateTimeZone.forID(timeZoneID);
157         }
158         LocalDate startInstant = new LocalDate(currentTime, dateTimeZone);
159         LocalDate endInstant = new LocalDate(millisAtEnd, dateTimeZone);
160         Days days = Days.daysBetween(startInstant, endInstant);
161         return days.getDays();
162     }
163 
164     @Test
165     public void checkCleanupForBasicDailyRollover() {
166         cp.maxHistory(20).simulatedNumberOfPeriods(20 * 3).startInactivity(0).numInactivityPeriods(0);
167         generateDailyRolloverAndCheckFileCount(cp);
168     }
169 
170     @Test
171     public void checkCleanupForBasicDailyRolloverWithSizeCap() {
172         long bytesOutputPerPeriod = 15984;
173         int sizeInUnitsOfBytesPerPeriod = 2;
174 
175         cp.maxHistory(5).simulatedNumberOfPeriods(10).sizeCap(sizeInUnitsOfBytesPerPeriod * bytesOutputPerPeriod + 1000);
176         generateDailyRollover(cp);
177         StatusPrinter.print(context);
178         checkFileCount(sizeInUnitsOfBytesPerPeriod + 1);
179     }
180 
181     @Test
182     public void checkThatSmallTotalSizeCapLeavesAtLeastOneArhcive() {
183         long WED_2016_03_23_T_131345_CET = WED_2016_03_23_T_230705_CET - 10 * CoreConstants.MILLIS_IN_ONE_HOUR;
184 
185         // long bytesOutputPerPeriod = 15984;
186 
187         cp = new ConfigParameters(WED_2016_03_23_T_131345_CET);
188         final int verySmallCapSize = 1;
189         cp.maxHistory(5).simulatedNumberOfPeriods(3).sizeCap(verySmallCapSize);
190         generateDailyRollover(cp);
191         StatusPrinter.print(context);
192         checkFileCountAtMost(1);
193        
194     }
195 
196     @Test
197     public void checkCleanupForBasicDailyRolloverWithMaxSize() {
198         cp.maxHistory(6).simulatedNumberOfPeriods(30).startInactivity(10).numInactivityPeriods(1);
199         generateDailyRolloverAndCheckFileCount(cp);
200     }
201 
202     // Since the duration of a month (in seconds) varies from month to month, tests with inactivity period must
203     // be conducted with daily rollover not monthly
204     @Test
205     public void checkCleanupForDailyRollover_15Periods() {
206         cp.maxHistory(5).simulatedNumberOfPeriods(15).startInactivity(6).numInactivityPeriods(3);
207         generateDailyRolloverAndCheckFileCount(cp);
208     }
209 
210     @Test
211     public void checkCleanupForDailyRolloverWithInactivity_30Periods() {
212         // / -------
213         cp.maxHistory(2).simulatedNumberOfPeriods(30).startInactivity(3).numInactivityPeriods(1);
214         generateDailyRolloverAndCheckFileCount(cp);
215     }
216 
217     @Test
218     public void checkCleanupForDailyRolloverWithInactivity_10Periods() {
219         this.currentTime = THU_2016_03_17_T_230330_CET;
220         cp.maxHistory(6).simulatedNumberOfPeriods(10).startInactivity(2).numInactivityPeriods(2);
221         generateDailyRolloverAndCheckFileCount(cp);
222     }
223 
224     @Test
225     public void checkCleanupForDailyRolloverWithSecondPhase() {
226         slashCount = computeSlashCount(DAILY_DATE_PATTERN);
227         int maxHistory = 5;
228         String fileNamePattern = randomOutputDir + "clean-%d{" + DAILY_DATE_PATTERN + "}.txt";
229 
230         ConfigParameters cp0 = new ConfigParameters(currentTime).maxHistory(maxHistory).fileNamePattern(fileNamePattern)
231                         .simulatedNumberOfPeriods(maxHistory * 2);
232         long endTime = logOverMultiplePeriods(cp0);
233 
234         ConfigParameters cp1 = new ConfigParameters(endTime + MILLIS_IN_DAY * 10).maxHistory(maxHistory).fileNamePattern(fileNamePattern)
235                         .simulatedNumberOfPeriods(maxHistory);
236         logOverMultiplePeriods(cp1);
237         checkFileCount(expectedCountWithoutFolders(maxHistory));
238     }
239 
240     @Test
241     public void dailyRolloverWithCronologPattern() {
242         this.slashCount = computeSlashCount(DAILY_CRONOLOG_DATE_PATTERN);
243         String fileNamePattern = randomOutputDir + "/%d{" + DAILY_CRONOLOG_DATE_PATTERN + "}/clean.txt.zip";
244         cp.maxHistory(8).fileNamePattern(fileNamePattern).simulatedNumberOfPeriods(8 * 3);
245         logOverMultiplePeriods(cp);
246         int expectedDirMin = 9 + slashCount;
247         int expectDirMax = expectedDirMin + 1 + 1;
248         expectedFileAndDirCount(9, expectedDirMin, expectDirMax);
249     }
250 
251     @Test
252     public void dailySizeBasedRolloverWithoutCap() {
253         SizeAndTimeBasedFNATP<Object> sizeAndTimeBasedFNATP = new SizeAndTimeBasedFNATP<Object>();
254         sizeAndTimeBasedFNATP.invocationGate = fixedRateInvocationGate;
255 
256         sizeAndTimeBasedFNATP.setMaxFileSize(new FileSize(10000));
257         tbfnatp = sizeAndTimeBasedFNATP;
258         this.slashCount = computeSlashCount(DAILY_DATE_PATTERN);
259         String fileNamePattern = randomOutputDir + "/%d{" + DAILY_DATE_PATTERN + "}-clean.%i.zip";
260         cp.maxHistory(5).fileNamePattern(fileNamePattern).simulatedNumberOfPeriods(5 * 4);
261         logOverMultiplePeriods(cp);
262         checkPatternCompliance(5 + 1 + slashCount, "\\d{4}-\\d{2}-\\d{2}-clean(\\.\\d)(.zip)?");
263     }
264 
265     @Test
266     public void dailySizeBasedRolloverWithSizeCap() {
267         SizeAndTimeBasedFNATP<Object> sizeAndTimeBasedFNATP = new SizeAndTimeBasedFNATP<Object>();
268         sizeAndTimeBasedFNATP.invocationGate = new FixedRateInvocationGate(ticksPerPeriod / 8);
269         long bytesPerPeriod = 17000;
270         long fileSize = (bytesPerPeriod) / 5;
271         int expectedFileCount = 10;
272         long sizeCap = expectedFileCount * fileSize;
273         sizeAndTimeBasedFNATP.setMaxFileSize(new FileSize(fileSize));
274         tbfnatp = sizeAndTimeBasedFNATP;
275         this.slashCount = computeSlashCount(DAILY_DATE_PATTERN);
276 
277         // 2016-03-05 00:14:39 CET
278         long simulatedTime = 1457133279186L;
279         ConfigParameters params = new ConfigParameters(simulatedTime);
280         String fileNamePattern = randomOutputDir + "/%d{" + DAILY_DATE_PATTERN + "}-clean.%i";
281         params.maxHistory(60).fileNamePattern(fileNamePattern).simulatedNumberOfPeriods(10).sizeCap(sizeCap);
282         logOverMultiplePeriods(params);
283 
284         List<File> foundFiles = findFilesByPattern("\\d{4}-\\d{2}-\\d{2}-clean(\\.\\d)");
285         Collections.sort(foundFiles, new Comparator<File>() {
286             public int compare(File f0, File f1) {
287                 String s0 = f0.getName().toString();
288                 String s1 = f1.getName().toString();
289                 return s0.compareTo(s1);
290             }
291         });
292         System.out.print(foundFiles);
293         StatusPrinter.print(context);
294         checkFileCount(expectedFileCount - 1);
295     }
296 
297     @Test
298     public void dailyChronologSizeBasedRollover() {
299         SizeAndTimeBasedFNATP<Object> sizeAndTimeBasedFNATP = new SizeAndTimeBasedFNATP<Object>();
300         sizeAndTimeBasedFNATP.setMaxFileSize(new FileSize(10000));
301         sizeAndTimeBasedFNATP.invocationGate = fixedRateInvocationGate;
302         tbfnatp = sizeAndTimeBasedFNATP;
303         slashCount = 1;
304         String fileNamePattern = randomOutputDir + "/%d{" + DAILY_DATE_PATTERN + "}/clean.%i.zip";
305         cp.maxHistory(5).fileNamePattern(fileNamePattern).simulatedNumberOfPeriods(5 * 3);
306         logOverMultiplePeriods(cp);
307         checkDirPatternCompliance(6);
308     }
309 
310     @Test
311     public void dailyChronologSizeBasedRolloverWithSecondPhase() {
312         SizeAndTimeBasedFNATP<Object> sizeAndTimeBasedFNATP = new SizeAndTimeBasedFNATP<Object>();
313         sizeAndTimeBasedFNATP.setMaxFileSize(new FileSize(10000));
314         sizeAndTimeBasedFNATP.invocationGate = fixedRateInvocationGate;
315         tbfnatp = sizeAndTimeBasedFNATP;
316         this.slashCount = 1;
317         String fileNamePattern = randomOutputDir + "/%d{" + DAILY_DATE_PATTERN + "}/clean.%i";
318         int maxHistory = 5;
319         cp.maxHistory(maxHistory).fileNamePattern(fileNamePattern).simulatedNumberOfPeriods(3);
320         long endTime = logOverMultiplePeriods(cp);
321 
322         int simulatedNumberOfPeriods = maxHistory * 4;
323         ConfigParameters cp1 = new ConfigParameters(endTime + MILLIS_IN_DAY * 7).maxHistory(maxHistory).fileNamePattern(fileNamePattern)
324                         .simulatedNumberOfPeriods(simulatedNumberOfPeriods);
325         logOverMultiplePeriods(cp1);
326         checkDirPatternCompliance(maxHistory + 1);
327     }
328 
329     void logTwiceAndStop(long currentTime, String fileNamePattern, int maxHistory) {
330         ConfigParameters params = new ConfigParameters(currentTime).fileNamePattern(fileNamePattern).maxHistory(maxHistory);
331         buildRollingFileAppender(params, DO_CLEAN_HISTORY_ON_START);
332         rfa.doAppend("Hello ----------------------------------------------------------" + new Date(currentTime));
333         currentTime += MILLIS_IN_DAY / 2;
334         add(tbrp.compressionFuture);
335         add(tbrp.cleanUpFuture);
336         waitForJobsToComplete();
337         tbrp.timeBasedFileNamingAndTriggeringPolicy.setCurrentTime(currentTime);
338         rfa.doAppend("Hello ----------------------------------------------------------" + new Date(currentTime));
339         rfa.stop();
340     }
341 
342     @Test
343     public void cleanHistoryOnStart() {
344         long simulatedTime = WED_2016_03_23_T_230705_CET;
345         System.out.println(new Date(simulatedTime));
346 
347         String fileNamePattern = randomOutputDir + "clean-%d{" + DAILY_DATE_PATTERN + "}.txt";
348         int maxHistory = 3;
349         for (int i = 0; i <= 5; i++) {
350             logTwiceAndStop(simulatedTime, fileNamePattern, maxHistory);
351             simulatedTime += MILLIS_IN_DAY;
352         }
353         StatusPrinter.print(context);
354         checkFileCount(expectedCountWithoutFolders(maxHistory));
355     }
356 
357     @Test
358     public void cleanHistoryOnStartWithDayPattern() {
359         long simulatedTime = WED_2016_03_23_T_230705_CET;
360         String fileNamePattern = randomOutputDir + "clean-%d{yyyy-MM-dd}.txt";
361         int maxHistory = 3;
362         for (int i = 0; i <= 5; i++) {
363             logTwiceAndStop(simulatedTime, fileNamePattern, maxHistory);
364             simulatedTime += MILLIS_IN_DAY;
365         }
366         StatusPrinter.print(context);
367         checkFileCount(expectedCountWithoutFolders(maxHistory));
368     }
369 
370     @Ignore
371     @Test
372     // this test assumes a high degree of collisions in the archived files. Every 24 hours, the archive
373     // belonging to the previous day will be overwritten. Given that logback goes 14 days (336 hours) in history
374     // to clean files on start up, it is bound to delete more recent files. It is not logback's responsibility
375     // to cater for such degenerate cases.
376     public void cleanHistoryOnStartWithHourPattern() {
377         long now = this.currentTime;
378         String fileNamePattern = randomOutputDir + "clean-%d{HH}.txt";
379         int maxHistory = 3;
380         for (int i = 0; i <= 5; i++) {
381             logTwiceAndStop(now, fileNamePattern, maxHistory);
382             now = now + MILLIS_IN_HOUR;
383         }
384         StatusPrinter.print(context);
385         checkFileCount(expectedCountWithoutFolders(maxHistory));
386     }
387 
388     int expectedCountWithoutFolders(int maxHistory) {
389         return maxHistory + 1;
390     }
391 
392     int expectedCountWithFolders(int maxHistory, boolean withExtraFolder) {
393         int numLogFiles = (maxHistory + 1);
394         int numLogFilesAndFolders = numLogFiles * 2;
395         int result = numLogFilesAndFolders + slashCount;
396         if (withExtraFolder)
397             result += 1;
398         return result;
399     }
400 
401     void buildRollingFileAppender(ConfigParameters cp, boolean cleanHistoryOnStart) {
402         rfa.setContext(context);
403         rfa.setEncoder(encoder);
404         tbrp.setContext(context);
405         tbrp.setFileNamePattern(cp.fileNamePattern);
406         tbrp.setMaxHistory(cp.maxHistory);
407         tbrp.setTotalSizeCap(new FileSize(cp.sizeCap));
408         tbrp.setParent(rfa);
409         tbrp.setCleanHistoryOnStart(cleanHistoryOnStart);
410         tbrp.timeBasedFileNamingAndTriggeringPolicy = tbfnatp;
411         tbrp.timeBasedFileNamingAndTriggeringPolicy.setCurrentTime(cp.simulatedTime);
412         tbrp.start();
413         rfa.setRollingPolicy(tbrp);
414         rfa.start();
415     }
416 
417     boolean DO_CLEAN_HISTORY_ON_START = true;
418     boolean DO_NOT_CLEAN_HISTORY_ON_START = false;
419 
420     long logOverMultiplePeriods(ConfigParameters cp) {
421 
422         buildRollingFileAppender(cp, DO_NOT_CLEAN_HISTORY_ON_START);
423 
424         int runLength = cp.simulatedNumberOfPeriods * ticksPerPeriod;
425         int startInactivityIndex = cp.startInactivity * ticksPerPeriod;
426         int endInactivityIndex = startInactivityIndex + cp.numInactivityPeriods * ticksPerPeriod;
427         long tickDuration = cp.periodDurationInMillis / ticksPerPeriod;
428 
429         System.out.println("cp.periodDurationInMillis=" + cp.periodDurationInMillis + ", tickDuration=:" + tickDuration + ", runLength=" + runLength);
430         for (int i = 0; i <= runLength; i++) {
431             Date currentDate = new Date(tbrp.timeBasedFileNamingAndTriggeringPolicy.getCurrentTime());
432             if (i < startInactivityIndex || i > endInactivityIndex) {
433                 StringBuilder sb = new StringBuilder("Hello");
434                 String currentDateStr = currentDate.toString();
435                 String iAsString = Integer.toString(i);
436                 sb.append(currentDateStr);
437                 SpacePadder.spacePad(sb, 66 + (3 - iAsString.length() - currentDateStr.length()));
438                 sb.append(iAsString);
439                 rfa.doAppend(sb.toString());
440             } 
441 
442             tbrp.timeBasedFileNamingAndTriggeringPolicy.setCurrentTime(addTime(tbrp.timeBasedFileNamingAndTriggeringPolicy.getCurrentTime(), tickDuration));
443             
444             add(tbrp.compressionFuture);
445             add(tbrp.cleanUpFuture);
446             waitForJobsToComplete();
447         }
448         
449         
450         try {
451             Thread.sleep(100);
452         } catch (InterruptedException e) {
453             // TODO Auto-generated catch block
454             e.printStackTrace();
455         }
456         rfa.stop();
457 
458         System.out.println("Current time at end of loop: "+new Date(tbrp.timeBasedFileNamingAndTriggeringPolicy.getCurrentTime()));
459         return tbrp.timeBasedFileNamingAndTriggeringPolicy.getCurrentTime();
460     }
461 
462     void fillWithChar(StringBuffer sb, char c, int count) {
463         for (int i = 0; i < count; i++) {
464             sb.append(c);
465         }
466     }
467 
468     boolean extraFolder(int numPeriods, int periodsPerEra, int beginPeriod, int maxHistory) {
469         int valueOfLastMonth = ((beginPeriod) + numPeriods) % periodsPerEra;
470         return (valueOfLastMonth < maxHistory);
471     }
472 
473     long addTime(long time, long timeToWait) {
474         return time + timeToWait;
475     }
476 
477     void expectedFileAndDirCount(int expectedFileAndDirCount, int expectedDirCountMin, int expectedDirCountMax) {
478         File dir = new File(randomOutputDir);
479         List<File> fileList = new ArrayList<File>();
480         findFilesInFolderRecursivelyByPatterMatch(dir, fileList, "clean");
481         List<File> dirList = new ArrayList<File>();
482         findAllFoldersInFolderRecursively(dir, dirList);
483         String msg = "expectedDirCountMin=" + expectedDirCountMin + ", expectedDirCountMax=" + expectedDirCountMax + " actual value=" + dirList.size();
484         assertTrue(msg, expectedDirCountMin <= dirList.size() && dirList.size() <= expectedDirCountMax);
485     }
486 
487     void checkFileCount(int expectedCount) {
488         File dir = new File(randomOutputDir);
489         List<File> fileList = new ArrayList<File>();
490         findAllDirsOrStringContainsFilesRecursively(dir, fileList, "clean");
491         assertEquals(expectedCount, fileList.size());
492     }
493 
494     void checkFileCountAtMost(int expectedCount) {
495         File dir = new File(randomOutputDir);
496         List<File> fileList = new ArrayList<File>();
497         findAllDirsOrStringContainsFilesRecursively(dir, fileList, "clean");
498         int fileListSize = fileList.size();
499         
500         assertTrue("file list size "+ fileListSize+", expectedCount="+expectedCount, fileListSize <= expectedCount);
501     }
502     
503     int expectedCountWithoutFoldersWithInactivity(int maxHistory, int totalPeriods, int endOfInactivity) {
504         int availableHistory = (totalPeriods + 1) - endOfInactivity;
505         int actualHistory = Math.min(availableHistory, maxHistory + 1);
506         return actualHistory;
507     }
508 
509     void genericFindMatching(final FileMatchFunction matchFunc, File dir, List<File> fileList, final String pattern, boolean includeDirs) {
510         if (dir.isDirectory()) {
511             File[] matchArray = dir.listFiles(new FileFilter() {
512                 public boolean accept(File f) {
513                     return f.isDirectory() || matchFunc.match(f, pattern);
514                 }
515             });
516             for (File f : matchArray) {
517                 if (f.isDirectory()) {
518                     if (includeDirs)
519                         fileList.add(f);
520                     genericFindMatching(matchFunc, f, fileList, pattern, includeDirs);
521                 } else
522                     fileList.add(f);
523             }
524         }
525     }
526 
527     private void findAllFoldersInFolderRecursively(File dir, List<File> fileList) {
528         FileMatchFunction alwaysFalse = new FileMatchFunction() {
529             public boolean match(File f, String pattern) {
530                 return false;
531             }
532         };
533         genericFindMatching(alwaysFalse, dir, fileList, null, true);
534     }
535 
536     private void findAllDirsOrStringContainsFilesRecursively(File dir, List<File> fileList, String pattern) {
537         FileMatchFunction matchFunction = new FileMatchFunction() {
538             public boolean match(File f, String pattern) {
539                 return f.getName().contains(pattern);
540             }
541         };
542         genericFindMatching(matchFunction, dir, fileList, pattern, true);
543     }
544 
545     void findFilesInFolderRecursivelyByPatterMatch(File dir, List<File> fileList, String pattern) {
546         FileMatchFunction matchByPattern = new FileMatchFunction() {
547             public boolean match(File f, String pattern) {
548                 return f.getName().matches(pattern);
549             }
550         };
551         genericFindMatching(matchByPattern, dir, fileList, pattern, false);
552     }
553 
554     Set<String> groupByClass(List<File> fileList, String regex) {
555         Pattern p = Pattern.compile(regex);
556         Set<String> set = new HashSet<String>();
557         for (File f : fileList) {
558             String n = f.getName();
559             Matcher m = p.matcher(n);
560             m.matches();
561             int begin = m.start(1);
562             String reduced = n.substring(0, begin);
563             set.add(reduced);
564         }
565         return set;
566     }
567 
568     void checkPatternCompliance(int expectedClassCount, String regex) {
569         Set<String> set = findFilesByPatternClass(regex);
570         assertEquals(expectedClassCount, set.size());
571     }
572 
573     private List<File> findFilesByPattern(String regex) {
574         File dir = new File(randomOutputDir);
575         List<File> fileList = new ArrayList<File>();
576         findFilesInFolderRecursivelyByPatterMatch(dir, fileList, regex);
577         return fileList;
578     }
579 
580     private Set<String> findFilesByPatternClass(String regex) {
581         List<File> fileList = findFilesByPattern(regex);
582         Set<String> set = groupByClass(fileList, regex);
583         return set;
584     }
585 
586     void checkDirPatternCompliance(int expectedClassCount) {
587         File dir = new File(randomOutputDir);
588         List<File> fileList = new ArrayList<File>();
589         findAllFoldersInFolderRecursively(dir, fileList);
590         for (File f : fileList) {
591             assertTrue(f.list().length >= 1);
592         }
593         assertEquals(expectedClassCount, fileList.size());
594     }
595 }