1 if (typeof JSDOC
== "undefined") JSDOC
= {};
4 @class Search a {@link JSDOC.TextStream} for language tokens.
6 JSDOC
.TokenReader
= function() {
8 this.keepWhite
= false;
9 this.keepComments
= false;
15 JSDOC
.TokenReader
.prototype.tokenize
= function(/**JSDOC.TextStream*/stream
) {
17 /**@ignore*/ tokens
.last
= function() { return tokens
[tokens
.length
-1]; }
18 /**@ignore*/ tokens
.lastSym
= function() {
19 for (var i
= tokens
.length
-1; i
>= 0; i
--) {
20 if (!(tokens
[i
].is("WHIT") || tokens
[i
].is("COMM"))) return tokens
[i
];
24 while (!stream
.look().eof
) {
25 if (this.read_mlcomment(stream
, tokens
)) continue;
26 if (this.read_slcomment(stream
, tokens
)) continue;
27 if (this.read_dbquote(stream
, tokens
)) continue;
28 if (this.read_snquote(stream
, tokens
)) continue;
29 if (this.read_regx(stream
, tokens
)) continue;
30 if (this.read_numb(stream
, tokens
)) continue;
31 if (this.read_punc(stream
, tokens
)) continue;
32 if (this.read_newline(stream
, tokens
)) continue;
33 if (this.read_space(stream
, tokens
)) continue;
34 if (this.read_word(stream
, tokens
)) continue;
36 // if execution reaches here then an error has happened
37 tokens
.push(new JSDOC
.Token(stream
.next(), "TOKN", "UNKNOWN_TOKEN"));
43 @returns {Boolean} Was the token found?
45 JSDOC
.TokenReader
.prototype.read_word
= function(/**JSDOC.TokenStream*/stream
, tokens
) {
47 while (!stream
.look().eof
&& JSDOC
.Lang
.isWordChar(stream
.look())) {
48 found
+= stream
.next();
56 if ((name
= JSDOC
.Lang
.keyword(found
))) tokens
.push(new JSDOC
.Token(found
, "KEYW", name
));
57 else tokens
.push(new JSDOC
.Token(found
, "NAME", "NAME"));
63 @returns {Boolean} Was the token found?
65 JSDOC
.TokenReader
.prototype.read_punc
= function(/**JSDOC.TokenStream*/stream
, tokens
) {
68 while (!stream
.look().eof
&& JSDOC
.Lang
.punc(found
+stream
.look())) {
69 found
+= stream
.next();
76 tokens
.push(new JSDOC
.Token(found
, "PUNC", JSDOC
.Lang
.punc(found
)));
82 @returns {Boolean} Was the token found?
84 JSDOC
.TokenReader
.prototype.read_space
= function(/**JSDOC.TokenStream*/stream
, tokens
) {
87 while (!stream
.look().eof
&& JSDOC
.Lang
.isSpace(stream
.look())) {
88 found
+= stream
.next();
95 if (this.collapseWhite
) found
= " ";
96 if (this.keepWhite
) tokens
.push(new JSDOC
.Token(found
, "WHIT", "SPACE"));
102 @returns {Boolean} Was the token found?
104 JSDOC
.TokenReader
.prototype.read_newline
= function(/**JSDOC.TokenStream*/stream
, tokens
) {
107 while (!stream
.look().eof
&& JSDOC
.Lang
.isNewline(stream
.look())) {
108 found
+= stream
.next();
115 if (this.collapseWhite
) found
= "\n";
116 if (this.keepWhite
) tokens
.push(new JSDOC
.Token(found
, "WHIT", "NEWLINE"));
122 @returns {Boolean} Was the token found?
124 JSDOC
.TokenReader
.prototype.read_mlcomment
= function(/**JSDOC.TokenStream*/stream
, tokens
) {
125 if (stream
.look() == "/" && stream
.look(1) == "*") {
126 var found
= stream
.next(2);
128 while (!stream
.look().eof
&& !(stream
.look(-1) == "/" && stream
.look(-2) == "*")) {
129 found
+= stream
.next();
132 // to start doclet we allow /** or
/*** but not /**/ or
/****
133 if (/^\/\*\*([^\/]|\*[^*])/.test(found) && this.keepDocs) tokens.push(new JSDOC.Token(found, "COMM", "JSDOC"));
134 else if (this.keepComments) tokens.push(new JSDOC.Token(found, "COMM", "MULTI_LINE_COMM"));
141 @returns {Boolean} Was the token found?
143 JSDOC
.TokenReader
.prototype.read_slcomment
= function(/**JSDOC.TokenStream*/stream
, tokens
) {
146 (stream
.look() == "/" && stream.look(1) == "/" && (found
=stream
.next(2)))
148 (stream
.look() == "<" && stream
.look(1) == "!" && stream
.look(2) == "-" && stream
.look(3) == "-" && (found
=stream
.next(4)))
151 while (!stream
.look().eof
&& !JSDOC
.Lang
.isNewline(stream
.look())) {
152 found
+= stream
.next();
155 if (this.keepComments
) {
156 tokens
.push(new JSDOC
.Token(found
, "COMM", "SINGLE_LINE_COMM"));
164 @returns {Boolean} Was the token found?
166 JSDOC
.TokenReader
.prototype.read_dbquote
= function(/**JSDOC.TokenStream*/stream
, tokens
) {
167 if (stream
.look() == "\"") {
169 var string
= stream
.next();
171 while (!stream
.look().eof
) {
172 if (stream
.look() == "\\") {
173 if (JSDOC
.Lang
.isNewline(stream
.look(1))) {
176 } while (!stream
.look().eof
&& JSDOC
.Lang
.isNewline(stream
.look()));
180 string
+= stream
.next(2);
183 else if (stream
.look() == "\"") {
184 string
+= stream
.next();
185 tokens
.push(new JSDOC
.Token(string
, "STRN", "DOUBLE_QUOTE"));
189 string
+= stream
.next();
193 return false; // error! unterminated string
197 @returns {Boolean} Was the token found?
199 JSDOC
.TokenReader
.prototype.read_snquote
= function(/**JSDOC.TokenStream*/stream
, tokens
) {
200 if (stream
.look() == "'") {
202 var string
= stream
.next();
204 while (!stream
.look().eof
) {
205 if (stream
.look() == "\\") { // escape sequence
206 string
+= stream
.next(2);
208 else if (stream
.look() == "'") {
209 string
+= stream
.next();
210 tokens
.push(new JSDOC
.Token(string
, "STRN", "SINGLE_QUOTE"));
214 string
+= stream
.next();
218 return false; // error! unterminated string
222 @returns {Boolean} Was the token found?
224 JSDOC
.TokenReader
.prototype.read_numb
= function(/**JSDOC.TokenStream*/stream
, tokens
) {
225 if (stream
.look() === "0" && stream
.look(1) == "x") {
226 return this.read_hex(stream
, tokens
);
231 while (!stream
.look().eof
&& JSDOC
.Lang
.isNumber(found
+stream
.look())){
232 found
+= stream
.next();
239 if (/^0[0-7]/.test(found
)) tokens
.push(new JSDOC
.Token(found
, "NUMB", "OCTAL"));
240 else tokens
.push(new JSDOC
.Token(found
, "NUMB", "DECIMAL"));
245 requires("../lib/JSDOC/TextStream.js");
246 requires("../lib/JSDOC/Token.js");
247 requires("../lib/JSDOC/Lang.js");
249 plan(3, "testing JSDOC.TokenReader.prototype.read_numb");
252 var src = "function foo(num){while (num+8.0 >= 0x20 && num < 0777){}}";
253 var tr = new JSDOC.TokenReader();
254 var tokens = tr.tokenize(new JSDOC.TextStream(src));
256 var hexToken, octToken, decToken;
257 for (var i = 0; i < tokens.length; i++) {
258 if (tokens[i].name == "HEX_DEC") hexToken = tokens[i];
259 if (tokens[i].name == "OCTAL") octToken = tokens[i];
260 if (tokens[i].name == "DECIMAL") decToken = tokens[i];
264 is(decToken.data, "8.0", "decimal number is found in source.");
265 is(hexToken.data, "0x20", "hexdec number is found in source (issue #99).");
266 is(octToken.data, "0777", "octal number is found in source.");
270 @returns {Boolean} Was the token found?
272 JSDOC
.TokenReader
.prototype.read_hex
= function(/**JSDOC.TokenStream*/stream
, tokens
) {
273 var found
= stream
.next(2);
275 while (!stream
.look().eof
) {
276 if (JSDOC
.Lang
.isHexDec(found
) && !JSDOC
.Lang
.isHexDec(found
+stream
.look())) { // done
277 tokens
.push(new JSDOC
.Token(found
, "NUMB", "HEX_DEC"));
281 found
+= stream
.next();
288 @returns {Boolean} Was the token found?
290 JSDOC
.TokenReader
.prototype.read_regx
= function(/**JSDOC.TokenStream*/stream
, tokens
) {
298 !(last
= tokens
.lastSym()) // there is no last, the regex is the first symbol
303 && !last
.is("RIGHT_PAREN")
304 && !last
.is("RIGHT_BRACKET")
309 var regex
= stream
.next();
311 while (!stream
.look().eof
) {
312 if (stream
.look() == "\\") { // escape sequence
313 regex
+= stream
.next(2);
315 else if (stream
.look() == "/") {
316 regex
+= stream
.next();
318 while (/[gmi]/.test(stream
.look())) {
319 regex
+= stream
.next();
322 tokens
.push(new JSDOC
.Token(regex
, "REGX", "REGX"));
326 regex
+= stream
.next();
329 // error: unterminated regex