Skip to content

Commit cc6e15b

Browse files
romainmenkeromainmenke
authored
align serializers more with cssom (#1601)
1 parent 63986cf commit cc6e15b

6 files changed

Lines changed: 154 additions & 46 deletions

File tree

‎packages/css-tokenizer/CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
# Changes to CSS Tokenizer
22

3+
### Unreleased (patch)
4+
5+
- align serializers with CSSOM
6+
37
### 3.0.3
48

59
_October 25, 2024_

‎packages/css-tokenizer/dist/index.cjs‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎packages/css-tokenizer/dist/index.mjs‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎packages/css-tokenizer/src/util/mutations.ts‎

Lines changed: 44 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import { HYPHEN_MINUS } from '../code-points/code-points';
2-
import { isHexDigitCodePoint, isIdentCodePoint, isIdentStartCodePoint } from '../code-points/ranges';
1+
import { HYPHEN_MINUS, NULL, REPLACEMENT_CHARACTER } from '../code-points/code-points';
2+
import { isIdentCodePoint, isIdentStartCodePoint } from '../code-points/ranges';
33
import type { TokenDimension, TokenIdent } from '../interfaces/token';
44

55
/**
@@ -13,7 +13,7 @@ export function mutateIdent(ident: TokenIdent, newValue: string): void {
1313
codePoints.push(codePoint.codePointAt(0)!);
1414
}
1515

16-
const result = String.fromCodePoint(...ensureThatValueRoundTripsAsIdent(codePoints));
16+
const result = String.fromCodePoint(...serializeIdent(codePoints));
1717

1818
ident[1] = result;
1919
ident[4].value = newValue;
@@ -30,7 +30,7 @@ export function mutateUnit(ident: TokenDimension, newUnit: string): void {
3030
codePoints.push(codePoint.codePointAt(0)!);
3131
}
3232

33-
const escapedCodePoints = ensureThatValueRoundTripsAsIdent(codePoints);
33+
const escapedCodePoints = serializeIdent(codePoints);
3434
if (escapedCodePoints[0] === 101) { // `e`
3535
insertEscapedCodePoint(escapedCodePoints, 0, escapedCodePoints[0]);
3636
}
@@ -44,17 +44,30 @@ export function mutateUnit(ident: TokenDimension, newUnit: string): void {
4444
ident[4].unit = newUnit;
4545
}
4646

47-
function ensureThatValueRoundTripsAsIdent(codePoints: Array<number>): Array<number> {
47+
function serializeIdent(codePoints: Array<number>): Array<number> {
4848
let remainderStartIndex = 0;
4949

50-
if (codePoints[0] === HYPHEN_MINUS && codePoints[1] === HYPHEN_MINUS) {
50+
if (codePoints[0] === NULL) {
51+
codePoints.splice(
52+
0,
53+
1,
54+
REPLACEMENT_CHARACTER
55+
);
56+
57+
remainderStartIndex = 1;
58+
} else if (codePoints[0] === HYPHEN_MINUS && codePoints[1] === HYPHEN_MINUS) {
5159
remainderStartIndex = 2;
5260
} else if (codePoints[0] === HYPHEN_MINUS && codePoints[1]) {
5361
remainderStartIndex = 2;
5462

5563
if (!isIdentStartCodePoint(codePoints[1])) {
5664
remainderStartIndex += insertEscapedCodePoint(codePoints, 1, codePoints[1]);
5765
}
66+
} else if (codePoints[0] === HYPHEN_MINUS && !codePoints[1]) {
67+
return [
68+
92, // `\` backslash
69+
codePoints[0]
70+
];
5871
} else if (isIdentStartCodePoint(codePoints[0])) {
5972
remainderStartIndex = 1;
6073
} else {
@@ -63,16 +76,38 @@ function ensureThatValueRoundTripsAsIdent(codePoints: Array<number>): Array<numb
6376
}
6477

6578
for (let i = remainderStartIndex; i < codePoints.length; i++) {
79+
if (codePoints[i] === NULL) {
80+
codePoints.splice(
81+
i,
82+
1,
83+
REPLACEMENT_CHARACTER
84+
);
85+
86+
i++;
87+
continue;
88+
}
89+
6690
if (isIdentCodePoint(codePoints[i])) {
6791
continue;
6892
}
6993

70-
i += insertEscapedCodePoint(codePoints, i, codePoints[i]);
94+
i += insertEscapedCharacter(codePoints, i, codePoints[i]);
7195
}
7296

7397
return codePoints;
7498
}
7599

100+
function insertEscapedCharacter(codePoints: Array<number>, index: number, codePoint: number): number {
101+
codePoints.splice(
102+
index,
103+
1,
104+
92, // `\` backslash
105+
codePoint
106+
);
107+
108+
return 1;
109+
}
110+
76111
function insertEscapedCodePoint(codePoints: Array<number>, index: number, codePoint: number): number {
77112
const hexRepresentation = codePoint.toString(16);
78113
const codePointsForHexRepresentation: Array<number> = [];
@@ -82,28 +117,13 @@ function insertEscapedCodePoint(codePoints: Array<number>, index: number, codePo
82117
codePointsForHexRepresentation.push(x.codePointAt(0)!);
83118
}
84119

85-
const next = codePoints[index + 1];
86-
if (
87-
(index === codePoints.length - 1) ||
88-
(next && isHexDigitCodePoint(next))
89-
) {
90-
codePoints.splice(
91-
index,
92-
1,
93-
92, // `\` backslash
94-
...codePointsForHexRepresentation,
95-
32, // ` ` space
96-
);
97-
98-
return 1 + codePointsForHexRepresentation.length;
99-
}
100-
101120
codePoints.splice(
102121
index,
103122
1,
104123
92, // `\` backslash
105124
...codePointsForHexRepresentation,
125+
32, // ` ` space
106126
);
107127

108-
return codePointsForHexRepresentation.length;
128+
return 1 + codePointsForHexRepresentation.length;
109129
}

‎packages/css-tokenizer/test/mutations/ident.mjs‎

Lines changed: 102 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,13 @@ import { mutateIdent, tokenizer, stringify } from '@csstools/css-tokenizer';
3636

3737
assert.deepEqual(
3838
token,
39-
['ident-token', 'foo\\2e bar', 0, 6, { value: 'foo.bar' }],
39+
['ident-token', 'foo\\.bar', 0, 6, { value: 'foo.bar' }],
4040
);
4141

4242
const raw = stringify(token);
4343
assert.deepEqual(
4444
raw,
45-
'foo\\2e bar',
45+
'foo\\.bar',
4646
);
4747

4848
const t = tokenizer({
@@ -52,7 +52,7 @@ import { mutateIdent, tokenizer, stringify } from '@csstools/css-tokenizer';
5252
assert.deepEqual(
5353
collectTokens(t),
5454
[
55-
['ident-token', 'foo\\2e bar', 0, 9, { value: 'foo.bar' }],
55+
['ident-token', 'foo\\.bar', 0, 7, { value: 'foo.bar' }],
5656
['EOF-token', '', -1, -1, undefined],
5757
],
5858
);
@@ -92,13 +92,13 @@ import { mutateIdent, tokenizer, stringify } from '@csstools/css-tokenizer';
9292

9393
assert.deepEqual(
9494
token,
95-
['ident-token', '\\2d ', 0, 2, { value: '-' }],
95+
['ident-token', '\\-', 0, 2, { value: '-' }],
9696
);
9797

9898
const raw = stringify(token);
9999
assert.deepEqual(
100100
raw,
101-
'\\2d ',
101+
'\\-',
102102
);
103103

104104
const t = tokenizer({
@@ -108,7 +108,7 @@ import { mutateIdent, tokenizer, stringify } from '@csstools/css-tokenizer';
108108
assert.deepEqual(
109109
collectTokens(t),
110110
[
111-
['ident-token', '\\2d ', 0, 3, { value: '-' }],
111+
['ident-token', '\\-', 0, 1, { value: '-' }],
112112
['EOF-token', '', -1, -1, undefined],
113113
],
114114
);
@@ -148,13 +148,13 @@ import { mutateIdent, tokenizer, stringify } from '@csstools/css-tokenizer';
148148

149149
assert.deepEqual(
150150
token,
151-
['ident-token', '--\\25other-prop', 0, 2, { value: '--%other-prop' }],
151+
['ident-token', '--\\%other-prop', 0, 2, { value: '--%other-prop' }],
152152
);
153153

154154
const raw = stringify(token);
155155
assert.deepEqual(
156156
raw,
157-
'--\\25other-prop',
157+
'--\\%other-prop',
158158
);
159159

160160
const t = tokenizer({
@@ -166,9 +166,9 @@ import { mutateIdent, tokenizer, stringify } from '@csstools/css-tokenizer';
166166
[
167167
[
168168
'ident-token',
169-
'--\\25other-prop',
169+
'--\\%other-prop',
170170
0,
171-
14,
171+
13,
172172
{ value: '--%other-prop' },
173173
],
174174
['EOF-token', '', -1, -1, undefined],
@@ -182,13 +182,13 @@ import { mutateIdent, tokenizer, stringify } from '@csstools/css-tokenizer';
182182

183183
assert.deepEqual(
184184
token,
185-
['ident-token', '--\\25 a-prop', 0, 2, { value: '--%a-prop' }],
185+
['ident-token', '--\\%a-prop', 0, 2, { value: '--%a-prop' }],
186186
);
187187

188188
const raw = stringify(token);
189189
assert.deepEqual(
190190
raw,
191-
'--\\25 a-prop',
191+
'--\\%a-prop',
192192
);
193193

194194
const t = tokenizer({
@@ -200,9 +200,9 @@ import { mutateIdent, tokenizer, stringify } from '@csstools/css-tokenizer';
200200
[
201201
[
202202
'ident-token',
203-
'--\\25 a-prop',
203+
'--\\%a-prop',
204204
0,
205-
11,
205+
9,
206206
{ value: '--%a-prop' },
207207
],
208208
['EOF-token', '', -1, -1, undefined],
@@ -252,7 +252,7 @@ import { mutateIdent, tokenizer, stringify } from '@csstools/css-tokenizer';
252252
token,
253253
[
254254
'ident-token',
255-
'-\\40webkit-width',
255+
'-\\40 webkit-width',
256256
0,
257257
2,
258258
{ value: '-@webkit-width' },
@@ -262,7 +262,7 @@ import { mutateIdent, tokenizer, stringify } from '@csstools/css-tokenizer';
262262
const raw = stringify(token);
263263
assert.deepEqual(
264264
raw,
265-
'-\\40webkit-width',
265+
'-\\40 webkit-width',
266266
);
267267

268268
const t = tokenizer({
@@ -274,9 +274,9 @@ import { mutateIdent, tokenizer, stringify } from '@csstools/css-tokenizer';
274274
[
275275
[
276276
'ident-token',
277-
'-\\40webkit-width',
277+
'-\\40 webkit-width',
278278
0,
279-
15,
279+
16,
280280
{ value: '-@webkit-width' },
281281
],
282282
['EOF-token', '', -1, -1, undefined],
@@ -311,3 +311,87 @@ import { mutateIdent, tokenizer, stringify } from '@csstools/css-tokenizer';
311311
],
312312
);
313313
}
314+
315+
{
316+
const token = ['ident-token', 'prop', 0, 2, { value: 'prop' }];
317+
mutateIdent(token, '-');
318+
319+
assert.deepEqual(
320+
token,
321+
['ident-token', '\\-', 0, 2, { value: '-' }],
322+
);
323+
324+
const raw = stringify(token);
325+
assert.deepEqual(
326+
raw,
327+
'\\-',
328+
);
329+
330+
const t = tokenizer({
331+
css: raw,
332+
});
333+
334+
assert.deepEqual(
335+
collectTokens(t),
336+
[
337+
['ident-token', '\\-', 0, 1, { value: '-' }],
338+
['EOF-token', '', -1, -1, undefined],
339+
],
340+
);
341+
}
342+
343+
{
344+
const token = ['ident-token', 'prop', 0, 2, { value: 'prop' }];
345+
mutateIdent(token, String.fromCodePoint(0x000));
346+
347+
assert.deepEqual(
348+
token,
349+
['ident-token', String.fromCodePoint(0xFFFD), 0, 2, { value: String.fromCodePoint(0x000) }],
350+
);
351+
352+
const raw = stringify(token);
353+
assert.deepEqual(
354+
raw,
355+
String.fromCodePoint(0xFFFD),
356+
);
357+
358+
const t = tokenizer({
359+
css: raw,
360+
});
361+
362+
assert.deepEqual(
363+
collectTokens(t),
364+
[
365+
['ident-token', String.fromCodePoint(0xFFFD), 0, 0, { value: String.fromCodePoint(0xFFFD) }],
366+
['EOF-token', '', -1, -1, undefined],
367+
],
368+
);
369+
}
370+
371+
{
372+
const token = ['ident-token', 'prop', 0, 2, { value: 'prop' }];
373+
mutateIdent(token, 'aa' + String.fromCodePoint(0x000));
374+
375+
assert.deepEqual(
376+
token,
377+
['ident-token', 'aa' + String.fromCodePoint(0xFFFD), 0, 2, { value: 'aa' + String.fromCodePoint(0x000) }],
378+
);
379+
380+
const raw = stringify(token);
381+
assert.deepEqual(
382+
raw,
383+
'aa' + String.fromCodePoint(0xFFFD),
384+
);
385+
386+
const t = tokenizer({
387+
css: raw,
388+
});
389+
390+
assert.deepEqual(
391+
collectTokens(t),
392+
[
393+
['ident-token', 'aa' + String.fromCodePoint(0xFFFD), 0, 2, { value: 'aa' + String.fromCodePoint(0xFFFD) }],
394+
['EOF-token', '', -1, -1, undefined],
395+
],
396+
);
397+
}

‎packages/media-query-list-parser/test/get-name/0001.mjs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ import { TokenType } from '@csstools/css-tokenizer';
4444

4545
assert.deepStrictEqual(
4646
feature.getNameToken(),
47-
['ident-token', 'min\\20width', -1, -1, { value: 'min width' }],
47+
['ident-token', 'min\\ width', -1, -1, { value: 'min width' }],
4848
);
4949
}
5050

@@ -90,7 +90,7 @@ import { TokenType } from '@csstools/css-tokenizer';
9090

9191
assert.deepStrictEqual(
9292
feature.getNameToken(),
93-
['ident-token', 'w\\20 dth', -1, -1, { value: 'w dth' }],
93+
['ident-token', 'w\\ dth', -1, -1, { value: 'w dth' }],
9494
);
9595
}
9696

0 commit comments

Comments
 (0)