1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 package ch.qos.cal10n.util;
23
24 import static org.junit.Assert.*;
25
26 import java.io.FileNotFoundException;
27 import java.io.FileReader;
28 import java.util.ArrayList;
29 import java.util.List;
30 import java.util.ResourceBundle;
31
32 import org.junit.Test;
33
34 import ch.qos.cal10n.Cal10nTestConstants;
35 import ch.qos.cal10n.util.Token.TokenType;
36
37
38
39
40 public class TokenStreamTest {
41
42
43 @Test
44 public void smoke() throws FileNotFoundException {
45 FileReader fr = new FileReader(Cal10nTestConstants.TEST_CLASSES + "/parser/smoke.properties");
46 TokenStream ts = new TokenStream(fr);
47 List<Token> tokenList = ts.tokenize();
48
49 List<Token> witness = new ArrayList<Token>();
50 witness.add(new Token(TokenType.KEY, "K0"));
51 witness.add(new Token(TokenType.SEPARATOR, "="));
52 witness.add(new Token(TokenType.VALUE, "V0"));
53 witness.add(Token.EOL);
54
55 witness.add(new Token(TokenType.KEY, "K1"));
56 witness.add(new Token(TokenType.SEPARATOR, "="));
57 witness.add(new Token(TokenType.VALUE, "V1"));
58 witness.add(Token.EOL);
59 assertEquals(witness, tokenList);
60 }
61
62 @Test
63 public void medium() throws FileNotFoundException {
64 FileReader fr = new FileReader(Cal10nTestConstants.TEST_CLASSES + "/parser/medium.properties");
65 TokenStream ts = new TokenStream(fr);
66 List<Token> tokenList = ts.tokenize();
67
68
69
70
71
72
73
74 List<Token> witness = new ArrayList<Token>();
75 witness.add(new Token(TokenType.KEY, "K0"));
76 witness.add(new Token(TokenType.SEPARATOR, "="));
77 witness.add(new Token(TokenType.VALUE, "V0 "));
78 witness.add(Token.TRAILING_BACKSLASH);
79 witness.add(Token.EOL);
80 witness.add(new Token(TokenType.VALUE, "X"));
81 witness.add(Token.EOL);
82 witness.add(Token.EOL);
83
84 witness.add(new Token(TokenType.KEY, "K1"));
85 witness.add(new Token(TokenType.SEPARATOR, "="));
86 witness.add(new Token(TokenType.VALUE, "V1"));
87 witness.add(Token.EOL);
88 assertEquals(witness, tokenList);
89 }
90
91 @Test
92 public void characters() throws FileNotFoundException {
93 FileReader fr = new FileReader(Cal10nTestConstants.TEST_CLASSES + "/parser/characters.properties");
94 TokenStream ts = new TokenStream(fr);
95 List<Token> tokenList = ts.tokenize();
96
97 List<Token> witness = new ArrayList<Token>();
98 witness.add(new Token(TokenType.KEY, "K0"));
99 witness.add(new Token(TokenType.SEPARATOR, "="));
100 witness.add(new Token(TokenType.VALUE, "a\nb"));
101 witness.add(Token.EOL);
102 witness.add(new Token(TokenType.KEY, "K1"));
103 witness.add(new Token(TokenType.SEPARATOR, "="));
104 witness.add(new Token(TokenType.VALUE, "a\u2297b\nc"));
105 witness.add(Token.EOL);
106 assertEquals(witness, tokenList);
107 }
108
109 @Test
110 public void resourceBundleWithSpecialCharacters() {
111 ResourceBundle rb = ResourceBundle.getBundle("parser/characters");
112 assertEquals("a\nb", rb.getObject("K0"));
113 assertEquals("a\u2297b\nc", rb.getObject("K1"));
114 }
115 }