View Javadoc
1   /**
2    * Logback: the reliable, generic, fast and flexible logging framework.
3    * Copyright (C) 1999-2015, QOS.ch. All rights reserved.
4    *
5    * This program and the accompanying materials are dual-licensed under
6    * either the terms of the Eclipse Public License v1.0 as published by
7    * the Eclipse Foundation
8    *
9    *   or (per the licensee's choosing)
10   *
11   * under the terms of the GNU Lesser General Public License version 2.1
12   * as published by the Free Software Foundation.
13   */
14  package ch.qos.logback.core.pattern.parser;
15  
16  import java.util.ArrayList;
17  import java.util.List;
18  
19  import ch.qos.logback.core.spi.ScanException;
20  import org.junit.jupiter.api.Test;
21  
22  import ch.qos.logback.core.pattern.util.AlmostAsIsEscapeUtil;
23  
24  import static org.junit.jupiter.api.Assertions.assertEquals;
25  import static org.junit.jupiter.api.Assertions.fail;
26  
27  public class TokenStreamTest {
28  
29      @Test
30      public void testEmpty() throws ScanException {
31          try {
32              new TokenStream("").tokenize();
33              fail("empty string not allowed");
34          } catch (IllegalArgumentException e) {
35          }
36      }
37  
38      @Test
39      public void testSingleLiteral() throws ScanException {
40          List<Token> tl = new TokenStream("hello").tokenize();
41          List<Token> witness = new ArrayList<Token>();
42          witness.add(new Token(Token.LITERAL, "hello"));
43          assertEquals(witness, tl);
44      }
45  
46      @Test
47      public void testLiteralWithPercent() throws ScanException {
48          {
49              List<Token> tl = new TokenStream("hello\\%world").tokenize();
50  
51              List<Token> witness = new ArrayList<Token>();
52              witness.add(new Token(Token.LITERAL, "hello%world"));
53              assertEquals(witness, tl);
54          }
55          {
56              List<Token> tl = new TokenStream("hello\\%").tokenize();
57              List<Token> witness = new ArrayList<Token>();
58              witness.add(new Token(Token.LITERAL, "hello%"));
59              assertEquals(witness, tl);
60          }
61  
62          {
63              List<Token> tl = new TokenStream("\\%").tokenize();
64              List<Token> witness = new ArrayList<Token>();
65              witness.add(new Token(Token.LITERAL, "%"));
66              assertEquals(witness, tl);
67          }
68      }
69  
70      @Test
71      public void testBasic() throws ScanException {
72  
73          // test "%c"
74          {
75              List<Token> tl = new TokenStream("%c").tokenize();
76              List<Token> witness = new ArrayList<Token>();
77              witness.add(Token.PERCENT_TOKEN);
78              witness.add(new Token(Token.SIMPLE_KEYWORD, "c"));
79              assertEquals(witness, tl);
80          }
81  
82          {
83              // test "xyz%-34c"
84              List<Token> tl = new TokenStream("%a%b").tokenize();
85              List<Token> witness = new ArrayList<Token>();
86              witness.add(Token.PERCENT_TOKEN);
87              witness.add(new Token(Token.SIMPLE_KEYWORD, "a"));
88              witness.add(Token.PERCENT_TOKEN);
89              witness.add(new Token(Token.SIMPLE_KEYWORD, "b"));
90              assertEquals(witness, tl);
91          }
92  
93          {
94              // test "xyz%-34c"
95              List<Token> tl = new TokenStream("xyz%-34c").tokenize();
96              List<Token> witness = new ArrayList<Token>();
97              witness.add(new Token(Token.LITERAL, "xyz"));
98              witness.add(Token.PERCENT_TOKEN);
99              witness.add(new Token(Token.FORMAT_MODIFIER, "-34"));
100             witness.add(new Token(Token.SIMPLE_KEYWORD, "c"));
101             assertEquals(witness, tl);
102         }
103     }
104 
105     @Test
106     public void testComplexNR() throws ScanException {
107         List<Token> tl = new TokenStream("%d{1234} [%34.-67toto] %n").tokenize();
108         List<Token> witness = new ArrayList<Token>();
109         witness.add(Token.PERCENT_TOKEN);
110         witness.add(new Token(Token.SIMPLE_KEYWORD, "d"));
111         List<String> ol = new ArrayList<String>();
112         ol.add("1234");
113         witness.add(new Token(Token.OPTION, ol));
114         witness.add(new Token(Token.LITERAL, " ["));
115         witness.add(Token.PERCENT_TOKEN);
116         witness.add(new Token(Token.FORMAT_MODIFIER, "34.-67"));
117         witness.add(new Token(Token.SIMPLE_KEYWORD, "toto"));
118         witness.add(new Token(Token.LITERAL, "] "));
119         witness.add(Token.PERCENT_TOKEN);
120         witness.add(new Token(Token.SIMPLE_KEYWORD, "n"));
121         assertEquals(witness, tl);
122     }
123 
124     @Test
125     public void testEmptyP() throws ScanException {
126         List<Token> tl = new TokenStream("()").tokenize();
127         List<Token> witness = new ArrayList<Token>();
128         witness.add(new Token(Token.LITERAL, "("));
129         witness.add(Token.RIGHT_PARENTHESIS_TOKEN);
130         assertEquals(witness, tl);
131     }
132 
133     @Test
134     public void testEmptyP2() throws ScanException {
135         List<Token> tl = new TokenStream("%()").tokenize();
136         List<Token> witness = new ArrayList<Token>();
137         witness.add(Token.PERCENT_TOKEN);
138         witness.add(Token.BARE_COMPOSITE_KEYWORD_TOKEN);
139         witness.add(Token.RIGHT_PARENTHESIS_TOKEN);
140         assertEquals(witness, tl);
141     }
142 
143     @Test
144     public void testEscape() throws ScanException {
145         {
146             List<Token> tl = new TokenStream("\\%").tokenize();
147             List<Token> witness = new ArrayList<Token>();
148             witness.add(new Token(Token.LITERAL, "%"));
149             assertEquals(witness, tl);
150         }
151 
152         {
153             List<Token> tl = new TokenStream("\\%\\(\\t\\)\\r\\n").tokenize();
154             List<Token> witness = new ArrayList<Token>();
155             witness.add(new Token(Token.LITERAL, "%(\t)\r\n"));
156             assertEquals(witness, tl);
157         }
158 
159         {
160             List<Token> tl = new TokenStream("\\\\%x").tokenize();
161             List<Token> witness = new ArrayList<Token>();
162             witness.add(new Token(Token.LITERAL, "\\"));
163             witness.add(Token.PERCENT_TOKEN);
164             witness.add(new Token(Token.SIMPLE_KEYWORD, "x"));
165             assertEquals(witness, tl);
166         }
167 
168         {
169             List<Token> tl = new TokenStream("%x\\)").tokenize();
170             List<Token> witness = new ArrayList<Token>();
171             witness.add(Token.PERCENT_TOKEN);
172             witness.add(new Token(Token.SIMPLE_KEYWORD, "x"));
173             witness.add(new Token(Token.LITERAL, ")"));
174             assertEquals(witness, tl);
175         }
176 
177         {
178             List<Token> tl = new TokenStream("%x\\_a").tokenize();
179             List<Token> witness = new ArrayList<Token>();
180             witness.add(Token.PERCENT_TOKEN);
181             witness.add(new Token(Token.SIMPLE_KEYWORD, "x"));
182             witness.add(new Token(Token.LITERAL, "a"));
183             assertEquals(witness, tl);
184         }
185         {
186             List<Token> tl = new TokenStream("%x\\_%b").tokenize();
187             List<Token> witness = new ArrayList<Token>();
188             witness.add(Token.PERCENT_TOKEN);
189             witness.add(new Token(Token.SIMPLE_KEYWORD, "x"));
190             witness.add(Token.PERCENT_TOKEN);
191             witness.add(new Token(Token.SIMPLE_KEYWORD, "b"));
192             assertEquals(witness, tl);
193         }
194     }
195 
196     @Test
197     public void testOptions() throws ScanException {
198         {
199             List<Token> tl = new TokenStream("%x{t}").tokenize();
200             List<Token> witness = new ArrayList<Token>();
201             witness.add(Token.PERCENT_TOKEN);
202             witness.add(new Token(Token.SIMPLE_KEYWORD, "x"));
203             List<String> ol = new ArrayList<String>();
204             ol.add("t");
205             witness.add(new Token(Token.OPTION, ol));
206             assertEquals(witness, tl);
207         }
208 
209         {
210             List<Token> tl = new TokenStream("%x{t,y}").tokenize();
211             List<Token> witness = new ArrayList<Token>();
212             witness.add(Token.PERCENT_TOKEN);
213             witness.add(new Token(Token.SIMPLE_KEYWORD, "x"));
214             List<String> ol = new ArrayList<String>();
215             ol.add("t");
216             ol.add("y");
217             witness.add(new Token(Token.OPTION, ol));
218             assertEquals(witness, tl);
219         }
220 
221         {
222             List<Token> tl = new TokenStream("%x{\"hello world.\", \"12y  \"}").tokenize();
223             List<Token> witness = new ArrayList<Token>();
224             witness.add(Token.PERCENT_TOKEN);
225             witness.add(new Token(Token.SIMPLE_KEYWORD, "x"));
226             List<String> ol = new ArrayList<String>();
227             ol.add("hello world.");
228             ol.add("12y  ");
229             witness.add(new Token(Token.OPTION, ol));
230             assertEquals(witness, tl);
231         }
232 
233         {
234             List<Token> tl = new TokenStream("%x{'opt}'}").tokenize();
235             List<Token> witness = new ArrayList<Token>();
236             witness.add(Token.PERCENT_TOKEN);
237             witness.add(new Token(Token.SIMPLE_KEYWORD, "x"));
238             List<String> ol = new ArrayList<String>();
239             ol.add("opt}");
240             witness.add(new Token(Token.OPTION, ol));
241             assertEquals(witness, tl);
242         }
243     }
244 
245     @Test
246     public void testSimpleP() throws ScanException {
247         List<Token> tl = new TokenStream("%(hello %class{.4?})").tokenize();
248         List<Token> witness = new ArrayList<Token>();
249         witness.add(Token.PERCENT_TOKEN);
250         witness.add(Token.BARE_COMPOSITE_KEYWORD_TOKEN);
251         witness.add(new Token(Token.LITERAL, "hello "));
252         witness.add(Token.PERCENT_TOKEN);
253         witness.add(new Token(Token.SIMPLE_KEYWORD, "class"));
254         List<String> ol = new ArrayList<String>();
255         ol.add(".4?");
256         witness.add(new Token(Token.OPTION, ol));
257         witness.add(Token.RIGHT_PARENTHESIS_TOKEN);
258         assertEquals(witness, tl);
259     }
260 
261     @Test
262     public void testSimpleP2() throws ScanException {
263         List<Token> tl = new TokenStream("X %a %-12.550(hello %class{.4?})").tokenize();
264         List<Token> witness = new ArrayList<Token>();
265         witness.add(new Token(Token.LITERAL, "X "));
266         witness.add(Token.PERCENT_TOKEN);
267         witness.add(new Token(Token.SIMPLE_KEYWORD, "a"));
268         witness.add(new Token(Token.LITERAL, " "));
269         witness.add(Token.PERCENT_TOKEN);
270         witness.add(new Token(Token.FORMAT_MODIFIER, "-12.550"));
271         witness.add(Token.BARE_COMPOSITE_KEYWORD_TOKEN);
272         witness.add(new Token(Token.LITERAL, "hello "));
273         witness.add(Token.PERCENT_TOKEN);
274         witness.add(new Token(Token.SIMPLE_KEYWORD, "class"));
275         List<String> ol = new ArrayList<String>();
276         ol.add(".4?");
277         witness.add(new Token(Token.OPTION, ol));
278         witness.add(Token.RIGHT_PARENTHESIS_TOKEN);
279         assertEquals(witness, tl);
280     }
281 
282     @Test
283     public void testMultipleRecursion() throws ScanException {
284         List<Token> tl = new TokenStream("%-1(%d %45(%class %file))").tokenize();
285         List<Token> witness = new ArrayList<Token>();
286         witness.add(Token.PERCENT_TOKEN);
287         witness.add(new Token(Token.FORMAT_MODIFIER, "-1"));
288         witness.add(Token.BARE_COMPOSITE_KEYWORD_TOKEN);
289         witness.add(Token.PERCENT_TOKEN);
290         witness.add(new Token(Token.SIMPLE_KEYWORD, "d"));
291         witness.add(new Token(Token.LITERAL, " "));
292         witness.add(Token.PERCENT_TOKEN);
293         witness.add(new Token(Token.FORMAT_MODIFIER, "45"));
294         witness.add(Token.BARE_COMPOSITE_KEYWORD_TOKEN);
295         witness.add(Token.PERCENT_TOKEN);
296         witness.add(new Token(Token.SIMPLE_KEYWORD, "class"));
297         witness.add(new Token(Token.LITERAL, " "));
298         witness.add(Token.PERCENT_TOKEN);
299         witness.add(new Token(Token.SIMPLE_KEYWORD, "file"));
300         witness.add(Token.RIGHT_PARENTHESIS_TOKEN);
301         witness.add(Token.RIGHT_PARENTHESIS_TOKEN);
302 
303         assertEquals(witness, tl);
304     }
305 
306     @Test
307     public void testNested() throws ScanException {
308         List<Token> tl = new TokenStream("%(%a%(%b))").tokenize();
309         List<Token> witness = new ArrayList<Token>();
310         witness.add(Token.PERCENT_TOKEN);
311         witness.add(Token.BARE_COMPOSITE_KEYWORD_TOKEN);
312         witness.add(Token.PERCENT_TOKEN);
313         witness.add(new Token(Token.SIMPLE_KEYWORD, "a"));
314         witness.add(Token.PERCENT_TOKEN);
315         witness.add(Token.BARE_COMPOSITE_KEYWORD_TOKEN);
316         witness.add(Token.PERCENT_TOKEN);
317         witness.add(new Token(Token.SIMPLE_KEYWORD, "b"));
318         witness.add(Token.RIGHT_PARENTHESIS_TOKEN);
319         witness.add(Token.RIGHT_PARENTHESIS_TOKEN);
320 
321         assertEquals(witness, tl);
322 
323     }
324 
325     @Test
326     public void testEscapedParanteheses() throws ScanException {
327         {
328             List<Token> tl = new TokenStream("\\(%h\\)").tokenize();
329             List<Token> witness = new ArrayList<Token>();
330             witness.add(new Token(Token.LITERAL, "("));
331             witness.add(Token.PERCENT_TOKEN);
332             witness.add(new Token(Token.SIMPLE_KEYWORD, "h"));
333             witness.add(new Token(Token.LITERAL, ")"));
334             assertEquals(witness, tl);
335         }
336         {
337             List<Token> tl = new TokenStream("(%h\\)").tokenize();
338             List<Token> witness = new ArrayList<Token>();
339             witness.add(new Token(Token.LITERAL, "("));
340             witness.add(Token.PERCENT_TOKEN);
341             witness.add(new Token(Token.SIMPLE_KEYWORD, "h"));
342             witness.add(new Token(Token.LITERAL, ")"));
343             assertEquals(witness, tl);
344         }
345         {
346             List<Token> tl = new TokenStream("%a(x\\)").tokenize();
347             List<Token> witness = new ArrayList<Token>();
348             witness.add(Token.PERCENT_TOKEN);
349             witness.add(new Token(Token.COMPOSITE_KEYWORD, "a"));
350             witness.add(new Token(Token.LITERAL, "x)"));
351             assertEquals(witness, tl);
352         }
353         {
354             List<Token> tl = new TokenStream("%a\\(x)").tokenize();
355             List<Token> witness = new ArrayList<Token>();
356             witness.add(Token.PERCENT_TOKEN);
357             witness.add(new Token(Token.SIMPLE_KEYWORD, "a"));
358             witness.add(new Token(Token.LITERAL, "(x"));
359             witness.add(new Token(Token.RIGHT_PARENTHESIS));
360 
361             assertEquals(witness, tl);
362         }
363     }
364 
365     @Test
366     public void testWindowsLikeBackSlashes() throws ScanException {
367         List<Token> tl = new TokenStream("c:\\hello\\world.%i", new AlmostAsIsEscapeUtil()).tokenize();
368 
369         List<Token> witness = new ArrayList<Token>();
370         witness.add(new Token(Token.LITERAL, "c:\\hello\\world."));
371         witness.add(Token.PERCENT_TOKEN);
372         witness.add(new Token(Token.SIMPLE_KEYWORD, "i"));
373         assertEquals(witness, tl);
374     }
375 
376     @Test
377     public void compositedKeyword() throws ScanException {
378         {
379             List<Token> tl = new TokenStream("%d(A)", new AlmostAsIsEscapeUtil()).tokenize();
380             List<Token> witness = new ArrayList<Token>();
381             witness.add(Token.PERCENT_TOKEN);
382             witness.add(new Token(Token.COMPOSITE_KEYWORD, "d"));
383             witness.add(new Token(Token.LITERAL, "A"));
384             witness.add(Token.RIGHT_PARENTHESIS_TOKEN);
385             assertEquals(witness, tl);
386         }
387         {
388             List<Token> tl = new TokenStream("a %subst(%b C)", new AlmostAsIsEscapeUtil()).tokenize();
389             List<Token> witness = new ArrayList<Token>();
390             witness.add(new Token(Token.LITERAL, "a "));
391             witness.add(Token.PERCENT_TOKEN);
392             witness.add(new Token(Token.COMPOSITE_KEYWORD, "subst"));
393             witness.add(Token.PERCENT_TOKEN);
394             witness.add(new Token(Token.SIMPLE_KEYWORD, "b"));
395             witness.add(new Token(Token.LITERAL, " C"));
396             witness.add(Token.RIGHT_PARENTHESIS_TOKEN);
397             assertEquals(witness, tl);
398         }
399     }
400 
401     @Test
402     public void compositedKeywordFollowedByOptions() throws ScanException {
403         {
404             List<Token> tl = new TokenStream("%d(A){o}", new AlmostAsIsEscapeUtil()).tokenize();
405             List<Token> witness = new ArrayList<Token>();
406             witness.add(Token.PERCENT_TOKEN);
407             witness.add(new Token(Token.COMPOSITE_KEYWORD, "d"));
408             witness.add(new Token(Token.LITERAL, "A"));
409             witness.add(Token.RIGHT_PARENTHESIS_TOKEN);
410             List<String> ol = new ArrayList<String>();
411             ol.add("o");
412             witness.add(new Token(Token.OPTION, ol));
413 
414             assertEquals(witness, tl);
415         }
416     }
417 }