001/**
002 * Logback: the reliable, generic, fast and flexible logging framework.
003 * Copyright (C) 1999-2015, QOS.ch. All rights reserved.
004 *
005 * This program and the accompanying materials are dual-licensed under
006 * either the terms of the Eclipse Public License v1.0 as published by
007 * the Eclipse Foundation
008 *
009 *   or (per the licensee's choosing)
010 *
011 * under the terms of the GNU Lesser General Public License version 2.1
012 * as published by the Free Software Foundation.
013 */
014package ch.qos.logback.core.pattern.parser;
015
016import static org.junit.Assert.assertEquals;
017import static org.junit.Assert.fail;
018
019import java.util.ArrayList;
020import java.util.List;
021
022import ch.qos.logback.core.spi.ScanException;
023import org.junit.Test;
024
025import ch.qos.logback.core.pattern.util.AlmostAsIsEscapeUtil;
026
027public class TokenStreamTest {
028
029    @Test
030    public void testEmpty() throws ScanException {
031        try {
032            new TokenStream("").tokenize();
033            fail("empty string not allowed");
034        } catch (IllegalArgumentException e) {
035        }
036    }
037
038    @Test
039    public void testSingleLiteral() throws ScanException {
040        List<Token> tl = new TokenStream("hello").tokenize();
041        List<Token> witness = new ArrayList<Token>();
042        witness.add(new Token(Token.LITERAL, "hello"));
043        assertEquals(witness, tl);
044    }
045
046    @Test
047    public void testLiteralWithPercent() throws ScanException {
048        {
049            List<Token> tl = new TokenStream("hello\\%world").tokenize();
050
051            List<Token> witness = new ArrayList<Token>();
052            witness.add(new Token(Token.LITERAL, "hello%world"));
053            assertEquals(witness, tl);
054        }
055        {
056            List<Token> tl = new TokenStream("hello\\%").tokenize();
057            List<Token> witness = new ArrayList<Token>();
058            witness.add(new Token(Token.LITERAL, "hello%"));
059            assertEquals(witness, tl);
060        }
061
062        {
063            List<Token> tl = new TokenStream("\\%").tokenize();
064            List<Token> witness = new ArrayList<Token>();
065            witness.add(new Token(Token.LITERAL, "%"));
066            assertEquals(witness, tl);
067        }
068    }
069
070    @Test
071    public void testBasic() throws ScanException {
072
073        // test "%c"
074        {
075            List<Token> tl = new TokenStream("%c").tokenize();
076            List<Token> witness = new ArrayList<Token>();
077            witness.add(Token.PERCENT_TOKEN);
078            witness.add(new Token(Token.SIMPLE_KEYWORD, "c"));
079            assertEquals(witness, tl);
080        }
081
082        {
083            // test "xyz%-34c"
084            List<Token> tl = new TokenStream("%a%b").tokenize();
085            List<Token> witness = new ArrayList<Token>();
086            witness.add(Token.PERCENT_TOKEN);
087            witness.add(new Token(Token.SIMPLE_KEYWORD, "a"));
088            witness.add(Token.PERCENT_TOKEN);
089            witness.add(new Token(Token.SIMPLE_KEYWORD, "b"));
090            assertEquals(witness, tl);
091        }
092
093        {
094            // test "xyz%-34c"
095            List<Token> tl = new TokenStream("xyz%-34c").tokenize();
096            List<Token> witness = new ArrayList<Token>();
097            witness.add(new Token(Token.LITERAL, "xyz"));
098            witness.add(Token.PERCENT_TOKEN);
099            witness.add(new Token(Token.FORMAT_MODIFIER, "-34"));
100            witness.add(new Token(Token.SIMPLE_KEYWORD, "c"));
101            assertEquals(witness, tl);
102        }
103    }
104
105    @Test
106    public void testComplexNR() throws ScanException {
107        List<Token> tl = new TokenStream("%d{1234} [%34.-67toto] %n").tokenize();
108        List<Token> witness = new ArrayList<Token>();
109        witness.add(Token.PERCENT_TOKEN);
110        witness.add(new Token(Token.SIMPLE_KEYWORD, "d"));
111        List<String> ol = new ArrayList<String>();
112        ol.add("1234");
113        witness.add(new Token(Token.OPTION, ol));
114        witness.add(new Token(Token.LITERAL, " ["));
115        witness.add(Token.PERCENT_TOKEN);
116        witness.add(new Token(Token.FORMAT_MODIFIER, "34.-67"));
117        witness.add(new Token(Token.SIMPLE_KEYWORD, "toto"));
118        witness.add(new Token(Token.LITERAL, "] "));
119        witness.add(Token.PERCENT_TOKEN);
120        witness.add(new Token(Token.SIMPLE_KEYWORD, "n"));
121        assertEquals(witness, tl);
122    }
123
124    @Test
125    public void testEmptyP() throws ScanException {
126        List<Token> tl = new TokenStream("()").tokenize();
127        List<Token> witness = new ArrayList<Token>();
128        witness.add(new Token(Token.LITERAL, "("));
129        witness.add(Token.RIGHT_PARENTHESIS_TOKEN);
130        assertEquals(witness, tl);
131    }
132
133    @Test
134    public void testEmptyP2() throws ScanException {
135        List<Token> tl = new TokenStream("%()").tokenize();
136        List<Token> witness = new ArrayList<Token>();
137        witness.add(Token.PERCENT_TOKEN);
138        witness.add(Token.BARE_COMPOSITE_KEYWORD_TOKEN);
139        witness.add(Token.RIGHT_PARENTHESIS_TOKEN);
140        assertEquals(witness, tl);
141    }
142
143    @Test
144    public void testEscape() throws ScanException {
145        {
146            List<Token> tl = new TokenStream("\\%").tokenize();
147            List<Token> witness = new ArrayList<Token>();
148            witness.add(new Token(Token.LITERAL, "%"));
149            assertEquals(witness, tl);
150        }
151
152        {
153            List<Token> tl = new TokenStream("\\%\\(\\t\\)\\r\\n").tokenize();
154            List<Token> witness = new ArrayList<Token>();
155            witness.add(new Token(Token.LITERAL, "%(\t)\r\n"));
156            assertEquals(witness, tl);
157        }
158
159        {
160            List<Token> tl = new TokenStream("\\\\%x").tokenize();
161            List<Token> witness = new ArrayList<Token>();
162            witness.add(new Token(Token.LITERAL, "\\"));
163            witness.add(Token.PERCENT_TOKEN);
164            witness.add(new Token(Token.SIMPLE_KEYWORD, "x"));
165            assertEquals(witness, tl);
166        }
167
168        {
169            List<Token> tl = new TokenStream("%x\\)").tokenize();
170            List<Token> witness = new ArrayList<Token>();
171            witness.add(Token.PERCENT_TOKEN);
172            witness.add(new Token(Token.SIMPLE_KEYWORD, "x"));
173            witness.add(new Token(Token.LITERAL, ")"));
174            assertEquals(witness, tl);
175        }
176
177        {
178            List<Token> tl = new TokenStream("%x\\_a").tokenize();
179            List<Token> witness = new ArrayList<Token>();
180            witness.add(Token.PERCENT_TOKEN);
181            witness.add(new Token(Token.SIMPLE_KEYWORD, "x"));
182            witness.add(new Token(Token.LITERAL, "a"));
183            assertEquals(witness, tl);
184        }
185        {
186            List<Token> tl = new TokenStream("%x\\_%b").tokenize();
187            List<Token> witness = new ArrayList<Token>();
188            witness.add(Token.PERCENT_TOKEN);
189            witness.add(new Token(Token.SIMPLE_KEYWORD, "x"));
190            witness.add(Token.PERCENT_TOKEN);
191            witness.add(new Token(Token.SIMPLE_KEYWORD, "b"));
192            assertEquals(witness, tl);
193        }
194    }
195
196    @Test
197    public void testOptions() throws ScanException {
198        {
199            List<Token> tl = new TokenStream("%x{t}").tokenize();
200            List<Token> witness = new ArrayList<Token>();
201            witness.add(Token.PERCENT_TOKEN);
202            witness.add(new Token(Token.SIMPLE_KEYWORD, "x"));
203            List<String> ol = new ArrayList<String>();
204            ol.add("t");
205            witness.add(new Token(Token.OPTION, ol));
206            assertEquals(witness, tl);
207        }
208
209        {
210            List<Token> tl = new TokenStream("%x{t,y}").tokenize();
211            List<Token> witness = new ArrayList<Token>();
212            witness.add(Token.PERCENT_TOKEN);
213            witness.add(new Token(Token.SIMPLE_KEYWORD, "x"));
214            List<String> ol = new ArrayList<String>();
215            ol.add("t");
216            ol.add("y");
217            witness.add(new Token(Token.OPTION, ol));
218            assertEquals(witness, tl);
219        }
220
221        {
222            List<Token> tl = new TokenStream("%x{\"hello world.\", \"12y  \"}").tokenize();
223            List<Token> witness = new ArrayList<Token>();
224            witness.add(Token.PERCENT_TOKEN);
225            witness.add(new Token(Token.SIMPLE_KEYWORD, "x"));
226            List<String> ol = new ArrayList<String>();
227            ol.add("hello world.");
228            ol.add("12y  ");
229            witness.add(new Token(Token.OPTION, ol));
230            assertEquals(witness, tl);
231        }
232
233        {
234            List<Token> tl = new TokenStream("%x{'opt}'}").tokenize();
235            List<Token> witness = new ArrayList<Token>();
236            witness.add(Token.PERCENT_TOKEN);
237            witness.add(new Token(Token.SIMPLE_KEYWORD, "x"));
238            List<String> ol = new ArrayList<String>();
239            ol.add("opt}");
240            witness.add(new Token(Token.OPTION, ol));
241            assertEquals(witness, tl);
242        }
243    }
244
245    @Test
246    public void testSimpleP() throws ScanException {
247        List<Token> tl = new TokenStream("%(hello %class{.4?})").tokenize();
248        List<Token> witness = new ArrayList<Token>();
249        witness.add(Token.PERCENT_TOKEN);
250        witness.add(Token.BARE_COMPOSITE_KEYWORD_TOKEN);
251        witness.add(new Token(Token.LITERAL, "hello "));
252        witness.add(Token.PERCENT_TOKEN);
253        witness.add(new Token(Token.SIMPLE_KEYWORD, "class"));
254        List<String> ol = new ArrayList<String>();
255        ol.add(".4?");
256        witness.add(new Token(Token.OPTION, ol));
257        witness.add(Token.RIGHT_PARENTHESIS_TOKEN);
258        assertEquals(witness, tl);
259    }
260
261    @Test
262    public void testSimpleP2() throws ScanException {
263        List<Token> tl = new TokenStream("X %a %-12.550(hello %class{.4?})").tokenize();
264        List<Token> witness = new ArrayList<Token>();
265        witness.add(new Token(Token.LITERAL, "X "));
266        witness.add(Token.PERCENT_TOKEN);
267        witness.add(new Token(Token.SIMPLE_KEYWORD, "a"));
268        witness.add(new Token(Token.LITERAL, " "));
269        witness.add(Token.PERCENT_TOKEN);
270        witness.add(new Token(Token.FORMAT_MODIFIER, "-12.550"));
271        witness.add(Token.BARE_COMPOSITE_KEYWORD_TOKEN);
272        witness.add(new Token(Token.LITERAL, "hello "));
273        witness.add(Token.PERCENT_TOKEN);
274        witness.add(new Token(Token.SIMPLE_KEYWORD, "class"));
275        List<String> ol = new ArrayList<String>();
276        ol.add(".4?");
277        witness.add(new Token(Token.OPTION, ol));
278        witness.add(Token.RIGHT_PARENTHESIS_TOKEN);
279        assertEquals(witness, tl);
280    }
281
282    @Test
283    public void testMultipleRecursion() throws ScanException {
284        List<Token> tl = new TokenStream("%-1(%d %45(%class %file))").tokenize();
285        List<Token> witness = new ArrayList<Token>();
286        witness.add(Token.PERCENT_TOKEN);
287        witness.add(new Token(Token.FORMAT_MODIFIER, "-1"));
288        witness.add(Token.BARE_COMPOSITE_KEYWORD_TOKEN);
289        witness.add(Token.PERCENT_TOKEN);
290        witness.add(new Token(Token.SIMPLE_KEYWORD, "d"));
291        witness.add(new Token(Token.LITERAL, " "));
292        witness.add(Token.PERCENT_TOKEN);
293        witness.add(new Token(Token.FORMAT_MODIFIER, "45"));
294        witness.add(Token.BARE_COMPOSITE_KEYWORD_TOKEN);
295        witness.add(Token.PERCENT_TOKEN);
296        witness.add(new Token(Token.SIMPLE_KEYWORD, "class"));
297        witness.add(new Token(Token.LITERAL, " "));
298        witness.add(Token.PERCENT_TOKEN);
299        witness.add(new Token(Token.SIMPLE_KEYWORD, "file"));
300        witness.add(Token.RIGHT_PARENTHESIS_TOKEN);
301        witness.add(Token.RIGHT_PARENTHESIS_TOKEN);
302
303        assertEquals(witness, tl);
304    }
305
306    @Test
307    public void testNested() throws ScanException {
308        List<Token> tl = new TokenStream("%(%a%(%b))").tokenize();
309        List<Token> witness = new ArrayList<Token>();
310        witness.add(Token.PERCENT_TOKEN);
311        witness.add(Token.BARE_COMPOSITE_KEYWORD_TOKEN);
312        witness.add(Token.PERCENT_TOKEN);
313        witness.add(new Token(Token.SIMPLE_KEYWORD, "a"));
314        witness.add(Token.PERCENT_TOKEN);
315        witness.add(Token.BARE_COMPOSITE_KEYWORD_TOKEN);
316        witness.add(Token.PERCENT_TOKEN);
317        witness.add(new Token(Token.SIMPLE_KEYWORD, "b"));
318        witness.add(Token.RIGHT_PARENTHESIS_TOKEN);
319        witness.add(Token.RIGHT_PARENTHESIS_TOKEN);
320
321        assertEquals(witness, tl);
322
323    }
324
325    @Test
326    public void testEscapedParanteheses() throws ScanException {
327        {
328            List<Token> tl = new TokenStream("\\(%h\\)").tokenize();
329            List<Token> witness = new ArrayList<Token>();
330            witness.add(new Token(Token.LITERAL, "("));
331            witness.add(Token.PERCENT_TOKEN);
332            witness.add(new Token(Token.SIMPLE_KEYWORD, "h"));
333            witness.add(new Token(Token.LITERAL, ")"));
334            assertEquals(witness, tl);
335        }
336        {
337            List<Token> tl = new TokenStream("(%h\\)").tokenize();
338            List<Token> witness = new ArrayList<Token>();
339            witness.add(new Token(Token.LITERAL, "("));
340            witness.add(Token.PERCENT_TOKEN);
341            witness.add(new Token(Token.SIMPLE_KEYWORD, "h"));
342            witness.add(new Token(Token.LITERAL, ")"));
343            assertEquals(witness, tl);
344        }
345        {
346            List<Token> tl = new TokenStream("%a(x\\)").tokenize();
347            List<Token> witness = new ArrayList<Token>();
348            witness.add(Token.PERCENT_TOKEN);
349            witness.add(new Token(Token.COMPOSITE_KEYWORD, "a"));
350            witness.add(new Token(Token.LITERAL, "x)"));
351            assertEquals(witness, tl);
352        }
353        {
354            List<Token> tl = new TokenStream("%a\\(x)").tokenize();
355            List<Token> witness = new ArrayList<Token>();
356            witness.add(Token.PERCENT_TOKEN);
357            witness.add(new Token(Token.SIMPLE_KEYWORD, "a"));
358            witness.add(new Token(Token.LITERAL, "(x"));
359            witness.add(new Token(Token.RIGHT_PARENTHESIS));
360
361            assertEquals(witness, tl);
362        }
363    }
364
365    @Test
366    public void testWindowsLikeBackSlashes() throws ScanException {
367        List<Token> tl = new TokenStream("c:\\hello\\world.%i", new AlmostAsIsEscapeUtil()).tokenize();
368
369        List<Token> witness = new ArrayList<Token>();
370        witness.add(new Token(Token.LITERAL, "c:\\hello\\world."));
371        witness.add(Token.PERCENT_TOKEN);
372        witness.add(new Token(Token.SIMPLE_KEYWORD, "i"));
373        assertEquals(witness, tl);
374    }
375
376    @Test
377    public void compositedKeyword() throws ScanException {
378        {
379            List<Token> tl = new TokenStream("%d(A)", new AlmostAsIsEscapeUtil()).tokenize();
380            List<Token> witness = new ArrayList<Token>();
381            witness.add(Token.PERCENT_TOKEN);
382            witness.add(new Token(Token.COMPOSITE_KEYWORD, "d"));
383            witness.add(new Token(Token.LITERAL, "A"));
384            witness.add(Token.RIGHT_PARENTHESIS_TOKEN);
385            assertEquals(witness, tl);
386        }
387        {
388            List<Token> tl = new TokenStream("a %subst(%b C)", new AlmostAsIsEscapeUtil()).tokenize();
389            List<Token> witness = new ArrayList<Token>();
390            witness.add(new Token(Token.LITERAL, "a "));
391            witness.add(Token.PERCENT_TOKEN);
392            witness.add(new Token(Token.COMPOSITE_KEYWORD, "subst"));
393            witness.add(Token.PERCENT_TOKEN);
394            witness.add(new Token(Token.SIMPLE_KEYWORD, "b"));
395            witness.add(new Token(Token.LITERAL, " C"));
396            witness.add(Token.RIGHT_PARENTHESIS_TOKEN);
397            assertEquals(witness, tl);
398        }
399    }
400
401    @Test
402    public void compositedKeywordFollowedByOptions() throws ScanException {
403        {
404            List<Token> tl = new TokenStream("%d(A){o}", new AlmostAsIsEscapeUtil()).tokenize();
405            List<Token> witness = new ArrayList<Token>();
406            witness.add(Token.PERCENT_TOKEN);
407            witness.add(new Token(Token.COMPOSITE_KEYWORD, "d"));
408            witness.add(new Token(Token.LITERAL, "A"));
409            witness.add(Token.RIGHT_PARENTHESIS_TOKEN);
410            List<String> ol = new ArrayList<String>();
411            ol.add("o");
412            witness.add(new Token(Token.OPTION, ol));
413
414            assertEquals(witness, tl);
415        }
416    }
417}