Skip to content
This repository was archived by the owner on Mar 23, 2024. It is now read-only.

Commit 1b497d6

Browse files
mdevilsmdevils
committed
Whitespace token
1 parent 7a85716 commit 1b497d6

15 files changed

Lines changed: 299 additions & 182 deletions

‎lib/js-file.js‎

Lines changed: 158 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -37,18 +37,20 @@ var JsFile = function(params) {
3737
this._lineBreaks = null;
3838
this._lines = this._source.split(/\r\n|\r|\n/);
3939

40+
var hasErrors = false;
4041
try {
4142
this._tree = parseJavaScriptSource(this._source, params.esprima, params.esprimaOptions);
4243
} catch (e) {
44+
hasErrors = true;
4345
this._parseErrors.push(e);
4446
}
4547

4648
// Lazy initialization
4749
this._scope = null;
4850

4951
this._tokens = this._buildTokenList(this._tree.tokens, this._tree.comments);
50-
this._addEOFToken();
51-
this._applyWhitespaceData(this._tokens, this._source);
52+
this._addEOFToken(hasErrors);
53+
this._tokens = this._addWhitespaceTokens(this._tokens, this._source);
5254

5355
this._setTokenIndexes();
5456

@@ -133,6 +135,50 @@ JsFile.prototype = {
133135
}, this);
134136
},
135137

138+
/**
139+
* Sets whitespace before specified token.
140+
*
141+
* @param {Object} token
142+
* @param {String} whitespace
143+
*/
144+
setWhitespaceBefore: function(token, whitespace) {
145+
var whitespaceToken = this.getPrevToken(token, {includeWhitespace: true});
146+
if (whitespaceToken && whitespaceToken.type === 'Whitespace') {
147+
if (whitespace === '') {
148+
this.removeToken(whitespaceToken);
149+
} else {
150+
whitespaceToken.value = whitespace;
151+
}
152+
} else {
153+
if (whitespace === '') {
154+
return;
155+
}
156+
var tokenIndex = this._tokens.indexOf(token);
157+
this._tokens.splice(tokenIndex, 0, {
158+
type: 'Whitespace',
159+
value: whitespace,
160+
isWhitespace: true
161+
});
162+
for (var i = tokenIndex, l = this._tokens.length; i < l; i++) {
163+
this._tokens[i]._tokenIndex = i;
164+
}
165+
}
166+
},
167+
168+
/**
169+
* Returns whitespace before specified token.
170+
*
171+
* @param {Object} token
172+
*/
173+
getWhitespaceBefore: function(token) {
174+
var whitespaceToken = this.getPrevToken(token, {includeWhitespace: true});
175+
if (whitespaceToken && whitespaceToken.type === 'Whitespace') {
176+
return whitespaceToken.value;
177+
} else {
178+
return '';
179+
}
180+
},
181+
136182
/**
137183
* Remove some entity (only one) from array with predicate
138184
*
@@ -150,7 +196,7 @@ JsFile.prototype = {
150196
},
151197

152198
/**
153-
* Remove token from token list
199+
* Remove token from token list.
154200
*
155201
* @param {Object} token
156202
*/
@@ -243,9 +289,15 @@ JsFile.prototype = {
243289
for (var i = 0, l = tokens.length; i < l; i++) {
244290
var token = tokens[i];
245291

292+
token._tokenIndex = i;
293+
294+
if (token.type === 'Whitespace') {
295+
continue;
296+
}
297+
246298
// tokens by range
247-
tokenRangeStartIndex[token.range[0]] = i;
248-
tokenRangeEndIndex[token.range[1]] = i;
299+
tokenRangeStartIndex[token.range[0]] = token;
300+
tokenRangeEndIndex[token.range[1]] = token;
249301

250302
// tokens by line
251303
var lineNumber = token.loc.start.line;
@@ -254,8 +306,6 @@ JsFile.prototype = {
254306
}
255307

256308
tokensByLineIndex[lineNumber].push(token);
257-
258-
token._tokenIndex = i;
259309
}
260310

261311
return {
@@ -271,8 +321,7 @@ JsFile.prototype = {
271321
* @returns {Object|null}
272322
*/
273323
getTokenByRangeStart: function(start) {
274-
var tokenIndex = this._tokenRangeStartIndex[start];
275-
return tokenIndex === undefined ? null : this._tokens[tokenIndex];
324+
return this._tokenRangeStartIndex[start] || null;
276325
},
277326

278327
/**
@@ -281,8 +330,7 @@ JsFile.prototype = {
281330
* @returns {Object|null}
282331
*/
283332
getTokenByRangeEnd: function(end) {
284-
var tokenIndex = this._tokenRangeEndIndex[end];
285-
return tokenIndex === undefined ? null : this._tokens[tokenIndex];
333+
return this._tokenRangeEndIndex[end] || null;
286334
},
287335

288336
/**
@@ -308,46 +356,53 @@ JsFile.prototype = {
308356
/**
309357
* Returns the first token for the file.
310358
*
359+
* @param {Option} [options]
360+
* @param {Boolean} [options.includeComments=false]
361+
* @param {Boolean} [options.includeWhitespace=false]
311362
* @returns {Object}
312363
*/
313-
getFirstToken: function() {
314-
return this._tokens[0];
364+
getFirstToken: function(options) {
365+
return this._getNextTokenFromIndex(0, options);
315366
},
316367

317368
/**
318369
* Returns the last token for the file.
319370
*
371+
* @param {Option} [options]
372+
* @param {Boolean} [options.includeComments=false]
373+
* @param {Boolean} [options.includeWhitespace=false]
320374
* @returns {Object}
321375
*/
322-
getLastToken: function() {
323-
return this._tokens[this._tokens.length - 1];
376+
getLastToken: function(options) {
377+
return this._getPrevTokenFromIndex(this._tokens.length - 1, options);
324378
},
325379

326380
/**
327-
* Returns the first token before the given.
381+
* Returns the first token before the given index.
328382
*
329-
* @param {Object} token
383+
* @param {Number} index
330384
* @param {Object} [options]
331385
* @param {Boolean} [options.includeComments=false]
386+
* @param {Boolean} [options.includeWhitespace=false]
332387
* @returns {Object|null}
333388
*/
334-
getPrevToken: function(token, options) {
335-
var index = token._tokenIndex - 1;
389+
_getPrevTokenFromIndex: function(index, options) {
336390
if (index < 0) {
337391
return null;
338392
}
339393

340-
if (options && options.includeComments) {
341-
return this._tokens[index] || null;
342-
}
343-
344394
do {
345-
if (!this._tokens[index]) {
395+
var prevToken = this._tokens[index];
396+
397+
if (!prevToken) {
346398
return null;
347399
}
348400

349-
if (!this._tokens[index].isComment) {
350-
return this._tokens[index];
401+
if (
402+
(!prevToken.isComment || (options && options.includeComments)) &&
403+
(!prevToken.isWhitespace || (options && options.includeWhitespace))
404+
) {
405+
return prevToken;
351406
}
352407
} while (--index >= 0);
353408

@@ -357,35 +412,60 @@ JsFile.prototype = {
357412
/**
358413
* Returns the first token after the given.
359414
*
360-
* @param {Object} token
415+
* @param {Number} index
361416
* @param {Object} [options]
362417
* @param {Boolean} [options.includeComments=false]
418+
* @param {Boolean} [options.includeWhitespace=false]
363419
* @returns {Object|null}
364420
*/
365-
getNextToken: function(token, options) {
366-
var index = token._tokenIndex + 1;
367-
421+
_getNextTokenFromIndex: function(index, options) {
368422
if (index >= this._tokens.length) {
369423
return null;
370424
}
371425

372-
if (options && options.includeComments) {
373-
return this._tokens[index] || null;
374-
}
375-
376426
do {
377-
if (!this._tokens[index]) {
427+
var nextToken = this._tokens[index];
428+
if (!nextToken) {
378429
return null;
379430
}
380431

381-
if (!this._tokens[index].isComment) {
382-
return this._tokens[index];
432+
if (
433+
(!nextToken.isComment || (options && options.includeComments)) &&
434+
(!nextToken.isWhitespace || (options && options.includeWhitespace))
435+
) {
436+
return nextToken;
383437
}
384438
} while (++index < this._tokens.length);
385439

386440
return null;
387441
},
388442

443+
/**
444+
* Returns the first token before the given.
445+
*
446+
* @param {Object} token
447+
* @param {Object} [options]
448+
* @param {Boolean} [options.includeComments=false]
449+
* @param {Boolean} [options.includeWhitespace=false]
450+
* @returns {Object|null}
451+
*/
452+
getPrevToken: function(token, options) {
453+
return this._getPrevTokenFromIndex(token._tokenIndex - 1, options);
454+
},
455+
456+
/**
457+
* Returns the first token after the given.
458+
*
459+
* @param {Object} token
460+
* @param {Object} [options]
461+
* @param {Boolean} [options.includeComments=false]
462+
* @param {Boolean} [options.includeWhitespace=false]
463+
* @returns {Object|null}
464+
*/
465+
getNextToken: function(token, options) {
466+
return this._getNextTokenFromIndex(token._tokenIndex + 1, options);
467+
},
468+
389469
/**
390470
* Returns the first token before the given which matches type (and value).
391471
*
@@ -559,7 +639,7 @@ JsFile.prototype = {
559639
typeIndex[type] = true;
560640
});
561641

562-
this.getTokens().forEach(function(token, index, tokens) {
642+
this._forEachToken(function(token, index, tokens) {
563643
if (typeIndex[token.type]) {
564644
cb(token, index, tokens);
565645
}
@@ -580,13 +660,30 @@ JsFile.prototype = {
580660
nameIndex[type] = true;
581661
});
582662

583-
this.getTokens().forEach(function(token, index, tokens) {
663+
this._forEachToken(function(token, index, tokens) {
584664
if (nameIndex.hasOwnProperty(token.value)) {
585665
cb(token, index, tokens);
586666
}
587667
});
588668
},
589669

670+
/**
671+
* Executes callback for each token in token list.
672+
*
673+
* @param {Function} cb
674+
* @private
675+
*/
676+
_forEachToken: function(cb) {
677+
var index = 0;
678+
var tokens = this._tokens;
679+
while (index < tokens.length) {
680+
var token = tokens[index];
681+
cb(token, index, tokens);
682+
index = token._tokenIndex;
683+
index++;
684+
}
685+
},
686+
590687
/**
591688
* Iterates tokens by type and value(s) from the token array.
592689
* Calls passed function for every matched token.
@@ -602,7 +699,7 @@ JsFile.prototype = {
602699
valueIndex[type] = true;
603700
});
604701

605-
this.getTokens().forEach(function(token, index, tokens) {
702+
this._forEachToken(function(token, index, tokens) {
606703
if (token.type === type && valueIndex[token.value]) {
607704
cb(token, index, tokens);
608705
}
@@ -805,8 +902,6 @@ JsFile.prototype = {
805902
for (var i = 0; i < this._tokens.length; i++) {
806903
var token = this._tokens[i];
807904

808-
result += token.whitespaceBefore;
809-
810905
switch (token.type) {
811906
// Line-comment: // ...
812907
case 'Line':
@@ -863,15 +958,17 @@ JsFile.prototype = {
863958
*
864959
* @private
865960
*/
866-
_addEOFToken: function() {
867-
var loc = {
868-
line: this._lines.length,
869-
column: this._lines[this._lines.length - 1].length
870-
};
961+
_addEOFToken: function(hasErrors) {
962+
var loc = hasErrors ?
963+
{line: 0, column: 0} :
964+
{
965+
line: this._lines.length,
966+
column: this._lines[this._lines.length - 1].length
967+
};
871968
this._tokens.push({
872969
type: 'EOF',
873970
value: '',
874-
range: [this._source.length, this._source.length + 1],
971+
range: hasErrors ? [0, 0] : [this._source.length, this._source.length + 1],
875972
loc: {start: loc, end: loc}
876973
});
877974
},
@@ -883,23 +980,29 @@ JsFile.prototype = {
883980
* @param {String} source
884981
* @private
885982
*/
886-
_applyWhitespaceData: function(tokens, source) {
983+
_addWhitespaceTokens: function(tokens, source) {
887984
var prevPos = 0;
985+
var result = [];
888986

889987
// For-loop for maximal speed.
890988
for (var i = 0; i < tokens.length; i++) {
891989
var token = tokens[i];
892990
var rangeStart = token.range[0];
893-
var whitespace;
894-
if (rangeStart === prevPos) {
895-
whitespace = '';
896-
} else {
897-
whitespace = source.substring(prevPos, rangeStart);
991+
if (rangeStart !== prevPos) {
992+
var whitespace = source.substring(prevPos, rangeStart);
993+
result.push({
994+
type: 'Whitespace',
995+
value: whitespace,
996+
isWhitespace: true
997+
});
898998
}
899999

900-
token.whitespaceBefore = whitespace;
1000+
result.push(token);
1001+
9011002
prevPos = token.range[1];
9021003
}
1004+
1005+
return result;
9031006
},
9041007

9051008
/**

0 commit comments

Comments
 (0)