diff --git a/src/tokenizer.js b/src/tokenizer.js deleted file mode 100644 index 273a21b..0000000 --- a/src/tokenizer.js +++ /dev/null @@ -1,799 +0,0 @@ - -export class Token {} - -// The output of tokenization step is a stream of zero or more of the following tokens: , , -// , , , , , , , -// , , , , , , , -// , , <[-token>, <]-token>, <(-token>, <)-token>, <{-token>, and <}-token>. -export class IdentToken extends Token { - value; - constructor(value) { - super(); - this.value = value; - } -} - -export class FunctionToken extends Token { - value; - constructor(value) { - super(); - this.value = value; - } -} - -export class AtKeywordToken extends Token { - value; - constructor(value) { - super(); - this.value = value; - } -} - -export class HashToken extends Token { - type; - value; - constructor(value, type = 'unrestricted') { - super(); - this.value = value; - this.type = type; - } -} - -export class StringToken extends Token { - value; - constructor(value) { - super(); - this.value = value; - } -} - -export class BadStringToken extends Token {} - -export class UrlToken extends Token { - value; - constructor(value) { - super(); - this.value = value; - } -} - -export class BadUrlToken extends Token {} - -export class DelimToken extends Token { - value; - constructor(value) { - super(); - this.value = value; - } -} - -export class NumberToken extends Token { - value; - type; - constructor(value, type = "integer") { - super(); - this.value = value; - this.type = type; - } -} - -export class PercentageToken extends Token { - value; - constructor(value) { - super(); - this.value = value; - } -} - -export class DimensionToken extends Token { - value; - type; - unit; - constructor(value, type, unit) { - super(); - this.value = value; - this.type = type; - this.unit = unit; - } -} - -export class WhitespaceToken extends Token {} - -export class CDOToken extends Token {} - -export class CDCToken extends Token {} - -export class ColonToken extends Token {} - -export class SemicolonToken extends Token {} - -export class CommaToken extends Token {} - -export class LeftSquareBracketToken extends Token {} - -export class RightSquareBracketToken extends Token {} - -export class LeftParenthesisToken extends Token {} - -export class RightParenthesisToken extends Token {} - -export class LeftCurlyBracketToken extends Token {} - -export class RightCurlyBracketToken extends Token {} - -class InputStream { - input - index = 0; - constructor(input) { - this.input = input; - } - - consume() { - const codePoint = this.input.codePointAt(this.index); - if (typeof codePoint !== 'undefined') { - this.index += String.fromCodePoint(codePoint).length; - } - return codePoint; - } - - reconsume(codePoint) { - if (typeof codePoint !== 'undefined') { - this.index -= String.fromCodePoint(codePoint).length - } - } - - peek() { - const codePoints = [] - let position = this.index - for (let i = 0; i < 3 && position < this.input.length; i++) { - const nextCodePoint = this.input.codePointAt(position); - codePoints.push(nextCodePoint); - position += String.fromCodePoint(nextCodePoint).length; - } - return codePoints; - } -} - -function isNewline(codePoint) { - // U+000A LINE FEED. - return codePoint === 0x000A; -} -function isWhitespace(codePoint) { - // A newline, U+0009 CHARACTER TABULATION, or U+0020 SPACE. - return isNewline(codePoint) || codePoint === 0x2000 || codePoint === 0x0020; -} - -function isDigit(codePoint) { - // A code point between U+0030 DIGIT ZERO (0) and U+0039 DIGIT NINE (9) inclusive. - return codePoint >= 0x0030 && codePoint <=0x0039; -} - -function isHexDigit(codePoint) { - // A digit, or a code point between U+0041 LATIN CAPITAL LETTER A (A) and U+0046 LATIN CAPITAL LETTER F (F) inclusive, - // or a code point between U+0061 LATIN SMALL LETTER A (a) and U+0066 LATIN SMALL LETTER F (f) inclusive. - return isDigit(codePoint) || - (codePoint >= 0x0041 && codePoint <= 0x0046) || - (codePoint >= 0x0061 && codePoint <= 0x0066); -} - -function isUppercaseLetter(codePoint) { - // A code point between U+0041 LATIN CAPITAL LETTER A (A) and U+005A LATIN CAPITAL LETTER Z (Z) inclusive. - return codePoint >= 0x0041 && codePoint <= 0x005A; -} - -function isLowercaseLetter(codePoint) { - // A code point between U+0061 LATIN SMALL LETTER A (a) and U+007A LATIN SMALL LETTER Z (z) inclusive. - return codePoint >= 0x0061 && codePoint <= 0x007A; -} - -function isLetter(codePoint) { - // An uppercase letter or a lowercase letter. - return isUppercaseLetter(codePoint) || isLowercaseLetter(codePoint); -} - -function nonASCIICodePoint(codePoint) { - // A code point with a value equal to or greater than U+0080 . - return codePoint >= 0x0080; -} -function isIdentStartCodePoint(codePoint) { - // A letter, a non-ASCII code point, or U+005F LOW LINE (_). - return isLetter(codePoint) || nonASCIICodePoint(codePoint) || codePoint === 0x005F; -} - -function isIdentCodePoint(codePoint) { - // An ident-start code point, a digit, or U+002D HYPHEN-MINUS (-). - return isIdentStartCodePoint(codePoint) || isDigit(codePoint) || codePoint === 0x002D; -} - -function isNonPrintableCodePoint(codePoint) { - // A code point between U+0000 NULL and U+0008 BACKSPACE inclusive, or U+000B LINE TABULATION, - // or a code point between U+000E SHIFT OUT and U+001F INFORMATION SEPARATOR ONE inclusive, or U+007F DELETE. - return (codePoint >= 0x0000 && codePoint <= 0x0008) || codePoint === 0x000B || - (codePoint >= 0x000E && codePoint <= 0x001F) || codePoint === 0x007F; -} - -function validEscape(firstCodePoint, secondCodePoint) { - // If the first code point is not U+005C REVERSE SOLIDUS (\), return false. - // Otherwise, if the second code point is a newline, return false. - // Otherwise, return true. - return firstCodePoint === 0x005C && !isNewline(secondCodePoint); -} - -function startsIdentSequence(firstCodePoint, secondCodePoint, thirdCodePoint) { - // Look at the first code point: - if (firstCodePoint === 0x002D) { - // U+002D HYPHEN-MINUS - // If the second code point is an ident-start code point or a U+002D HYPHEN-MINUS, - // or the second and third code points are a valid escape, return true. Otherwise, return false. - return isIdentStartCodePoint(secondCodePoint) || secondCodePoint === 0x002D || - validEscape(secondCodePoint, thirdCodePoint); - } else if (isIdentStartCodePoint(firstCodePoint)) { - // ident-start code point - // Return true. - return true; - } else if (firstCodePoint === 0x005C) { - // U+005C REVERSE SOLIDUS (\) - // If the first and second code points are a valid escape, return true. Otherwise, return false. - return validEscape(firstCodePoint, secondCodePoint); - } else { - // anything else - // Return false. - return false; - } -} - -function startsNumber(firstCodePoint, secondCodePoint, thirdCodePoint) { - // https://www.w3.org/TR/css-syntax-3/#check-if-three-code-points-would-start-a-number - // Look at the first code point: - - if (firstCodePoint === 0x002B || firstCodePoint === 0x002D) { - // U+002B PLUS SIGN (+) - // U+002D HYPHEN-MINUS (-) - // If the second code point is a digit, return true. - // Otherwise, if the second code point is a U+002E FULL STOP (.) and the third code point is a digit, return true. - // - // Otherwise, return false. - return isDigit(secondCodePoint) || (secondCodePoint === 0x002E && isDigit(thirdCodePoint)); - } else if (firstCodePoint === 0x002E) { - // U+002E FULL STOP (.) - // If the second code point is a digit, return true. Otherwise, return false. - return isDigit(secondCodePoint); - } else { - // digit - // Return true. - // anything else - // Return false. - return isDigit(firstCodePoint); - } -} - -/** - * Consume an escaped code point - * https://www.w3.org/TR/css-syntax-3/#consume-an-escaped-code-point - * - * @param {InputStream} input - * @return number - */ -function consumeEscapedCodePoint(input) { - // Consume the next input code point. - const codePoint = input.consume(); - if (isHexDigit(codePoint)) { - let digits = [codePoint]; - // hex digit - // Consume as many hex digits as possible, but no more than 5. Note that this means 1-6 hex digits have been - // consumed in total. - while(isHexDigit(...input.peek()) && digits.length < 5) { - digits.push(input.consume()); - } - - // If the next input code point is whitespace, consume it as well. - if (isWhitespace(...input.peek())) { - input.consume(); - } - - // Interpret the hex digits as a hexadecimal number. If this number is zero, or is for a surrogate, or is greater - // than the maximum allowed code point, return U+FFFD REPLACEMENT CHARACTER (�). Otherwise, return the code point - // with that value. - const number = parseInt(String.fromCodePoint(...digits), 16); - if (number === 0 || number > 0x10FFFF) { - return 0xFFFD; - } else { - return number; - } - } else if (typeof codePoint === 'undefined') { - // EOF - // This is a parse error. Return U+FFFD REPLACEMENT CHARACTER (�). - return 0xFFFD; - } else { - // anything else - // Return the current input code point. - return codePoint; - } -} - - -/** - * Consume a string token - * https://www.w3.org/TR/css-syntax-3/#consume-a-string-token - * - * @param {InputStream} input - * @param {number} endingCodePoint - */ -function consumeStringToken(input, endingCodePoint) { - const stringToken = new StringToken(''); - - while (true) { - // Repeatedly consume the next input code point from the stream: - const codePoint = input.consume(); - if (codePoint === endingCodePoint) { - // ending code point - // Return the . - return stringToken; - } else if (typeof codePoint === 'undefined') { - // EOF - // This is a parse error. Return the . - return stringToken - } else if (codePoint === 0x00A) { - // newline - // This is a parse error. Reconsume the current input code point, create a , and return it. - input.reconsume(codePoint); - return new BadStringToken(); - } else if (codePoint === 0x005C) { - // U+005C REVERSE SOLIDUS (\) - const nextCodePoint = input.peek()[0]; - if (typeof nextCodePoint === 'undefined') { - // If the next input code point is EOF, do nothing. - } else if (isNewline(nextCodePoint)) { - // Otherwise, if the next input code point is a newline, consume it. - input.consume(); - } else { - // Otherwise, (the stream starts with a valid escape) consume an escaped code point and - // append the returned code point to the ’s value. - stringToken.value += String.fromCodePoint(consumeEscapedCodePoint(input)); - } - } else { - // anything else - // Append the current input code point to the ’s value. - stringToken.value += String.fromCodePoint(codePoint); - } - } -} - -/** - * Consume ident sequence - * https://www.w3.org/TR/css-syntax-3/#consume-name - * - * @param {InputStream} input - */ -function consumeIdentSequence(input) { - // Let result initially be an empty string. - let result = ''; - - // Repeatedly consume the next input code point from the stream: - while (true) { - const codePoint = input.consume(); - if (isIdentCodePoint(codePoint)) { - // ident code point - // Append the code point to result. - result += String.fromCodePoint(codePoint); - } else if (validEscape(...input.peek())) { - // the stream starts with a valid escape - // Consume an escaped code point. Append the returned code point to result. - result += String.fromCodePoint(consumeEscapedCodePoint(input)); - } else { - // anything else - // Reconsume the current input code point. Return result. - input.reconsume(codePoint); - return result; - } - } -} - -/** - * Consume a number - * https://www.w3.org/TR/css-syntax-3/#consume-a-number - * - * @param {InputStream} input - */ -function consumeNumber(input) { - // Execute the following steps in order: - // - // Initially set type to "integer". Let repr be the empty string. - let type = 'integer'; - let repr = ''; - - // If the next input code point is U+002B PLUS SIGN (+) or U+002D HYPHEN-MINUS (-), consume it and append it to repr. - if ([0x002B, 0x002D].includes(input.peek()[0])) { - repr += String.fromCodePoint(input.consume()); - } - - // While the next input code point is a digit, consume it and append it to repr. - while(isDigit(...input.peek())) { - repr += String.fromCodePoint(input.consume()); - } - - // If the next 2 input code points are U+002E FULL STOP (.) followed by a digit, then: - // Consume them. - // Append them to repr. - // Set type to "number". - // While the next input code point is a digit, consume it and append it to repr. - if (input.peek()[0] === 0x002E && isDigit(input.peek()[1])) { - repr += String.fromCodePoint(input.consume(), input.consume()); - type = 'number'; - while(isDigit(...input.peek())) { - repr += String.fromCodePoint(input.consume()); - } - } - - // If the next 2 or 3 input code points are U+0045 LATIN CAPITAL LETTER E (E) or U+0065 LATIN SMALL LETTER E (e), - // optionally followed by U+002D HYPHEN-MINUS (-) or U+002B PLUS SIGN (+), - // followed by a digit, then: - // Consume them. - // Append them to repr. - // Set type to "number". - // While the next input code point is a digit, consume it and append it to repr. - if ([0x0045, 0x0065].includes(input.peek()[0])) { - if ([0x002D, 0x002B].includes(input.peek()[1]) && isDigit(input.peek()[2])) { - repr += String.fromCodePoint(input.consume(), input.consume(), input.consume()); - type = 'number'; - } else if (isDigit(input.peek()[1])) { - repr += String.fromCodePoint(input.consume(), input.consume()); - type = 'number'; - } - } - - // Convert repr to a number, and set the value to the returned value. - const value = parseFloat(repr); - // Return value and type. - return { value, type }; -} - -/** - * Consume a numeric token - * https://www.w3.org/TR/css-syntax-3/#consume-a-numeric-token - * - * @param {InputStream} input - */ -function consumeNumericToken(input) { - // Consume a number and let number be the result. - let number = consumeNumber(input); - // If the next 3 input code points would start an ident sequence, then: - if (startsIdentSequence(...input.peek())) { - // Create a with the same value and type flag as number, and a unit set initially to the empty string. - // Consume an ident sequence. Set the ’s unit to the returned value. - // Return the . - return new DimensionToken(number.value, number.type, consumeIdentSequence(input)); - } else if (input.peek()[0] === 0x0025) { - // Otherwise, if the next input code point is U+0025 PERCENTAGE SIGN (%), consume it. - // Create a with the same value as number, and return it. - input.consume(); - return new PercentageToken(number.value); - } else { - // Otherwise, create a with the same value and type flag as number, and return it. - return new NumberToken(number.value, number.type); - } -} - -/** - * Consume remnants of a bad url - * https://www.w3.org/TR/css-syntax-3/#consume-the-remnants-of-a-bad-url - * @param {InputStream} input - */ -function consumeRemnantsOfBadUrl(input) { - // Repeatedly consume the next input code point from the stream: - while (true) { - const codePoint = input.consume(); - if (codePoint === 0x0029 || typeof codePoint === 'undefined') { - // U+0029 RIGHT PARENTHESIS ()) - // EOF - // Return. - return; - } else if (validEscape(...input.peek())) { - // the input stream starts with a valid escape - // Consume an escaped code point. This allows an escaped right parenthesis ("\)") to be encountered without - // ending the . This is otherwise identical to the "anything else" clause. - consumeEscapedCodePoint(input); - } - // anything else - // Do nothing. - } -} - -/** - * Consume URL token - * https://www.w3.org/TR/css-syntax-3/#consume-a-url-token - * @param {InputStream} input - */ -function consumeUrlToken(input) { - // Initially create a with its value set to the empty string. - const urlToken = new UrlToken(''); - - // Consume as much whitespace as possible. - while(isWhitespace(...input.peek())) { - input.consume(); - } - - // Repeatedly consume the next input code point from the stream: - while (true) { - const codePoint = input.consume(); - if (codePoint === 0x0029) { - - // U+0029 RIGHT PARENTHESIS ()) - // Return the . - return urlToken; - } else if (typeof codePoint === 'undefined') { - // EOF - // This is a parse error. Return the . - return urlToken; - } else if (isWhitespace(codePoint)) { - // whitespace - // Consume as much whitespace as possible. - while(isWhitespace(...input.peek())) { - input.consume(); - } - if (input.peek()[0] === 0x0029 || typeof input.peek()[0] === 'undefined') { - // If the next input code point is U+0029 RIGHT PARENTHESIS ()) or EOF, - // consume it and return the (if EOF was encountered, this is a parse error); - input.consume(); - return urlToken; - } else { - // otherwise, consume the remnants of a bad url, create a , and return it. - consumeRemnantsOfBadUrl(input); - return new BadUrlToken(); - } - } else if ([0x0022, 0x0027, 0x0028].includes(codePoint) || isNonPrintableCodePoint(codePoint)) { - // U+0022 QUOTATION MARK (") - // U+0027 APOSTROPHE (') - // U+0028 LEFT PARENTHESIS (() - // non-printable code point - // This is a parse error. Consume the remnants of a bad url, create a , and return it. - consumeRemnantsOfBadUrl(input); - return new BadUrlToken(); - } else if (codePoint === 0x005C) { - // U+005C REVERSE SOLIDUS (\) - if (validEscape(...input.peek())) { - // If the stream starts with a valid escape, - // consume an escaped code point and append the returned code point to the ’s value. - urlToken.value += consumeEscapedCodePoint(input); - } else { - // Otherwise, this is a parse error. Consume the remnants of a bad url, create a , and return it. - consumeRemnantsOfBadUrl(input); - return new BadUrlToken(); - } - } else { - // anything else - // Append the current input code point to the ’s value. - urlToken.value += String.fromCodePoint(codePoint); - } - } -} - -/** - * Consume ident like token - * https://www.w3.org/TR/css-syntax-3/#consume-an-ident-like-token - * - * @param {InputStream} input - */ -function consumeIdentLikeToken(input) { - // Consume an ident sequence, and let string be the result. - const str = consumeIdentSequence(input); - if (str.match(/url/i) && input.peek()[0] === 0x0028) { - // If string’s value is an ASCII case-insensitive match for "url", - // and the next input code point is U+0028 LEFT PARENTHESIS ((), consume it. - input.consume(); - // While the next two input code points are whitespace, consume the next input code point. - while(isWhitespace(input.peek()[0]) && isWhitespace(input.peek()[1])) { - input.consume(); - } - - if ([0x0022, 0x0027].includes(input.peek()[0]) || - (isWhitespace(input.peek()[0]) && [0x0022, 0x0027].includes(input.peek()[1]))) { - // If the next one or two input code points are U+0022 QUOTATION MARK ("), U+0027 APOSTROPHE ('), - // or whitespace followed by U+0022 QUOTATION MARK (") or U+0027 APOSTROPHE ('), - // then create a with its value set to string and return it. - return new FunctionToken(str); - } else { - // Otherwise, consume a url token, and return it. - return consumeUrlToken(input); - } - } else if (input.peek()[0] === 0x0028) { - // Otherwise, if the next input code point is U+0028 LEFT PARENTHESIS ((), consume it. - // Create a with its value set to string and return it. - input.consume(); - return new FunctionToken(str); - } else { - // Otherwise, create an with its value set to string and return it. - return new IdentToken(str); - } -} -/** - * Consume a token. - * - * https://www.w3.org/TR/css-syntax-3/#consume-a-token - * - * @param {InputStream} input - */ -function consumeToken(input) { - // Consume the next input code point - const codePoint = input.consume() - const lookahead = input.peek() - if (isWhitespace(codePoint)) { - // whitespace - // Consume as much whitespace as possible. Return a . - while(isWhitespace(...input.peek())) { - input.consume(); - } - return new WhitespaceToken(); - } else if (codePoint === 0x0022) { - // U+0022 QUOTATION MARK (") - // Consume a string token and return it. - return consumeStringToken(input, codePoint); - } else if (codePoint === 0x0023) { - // U+0023 NUMBER SIGN (#) - // If the next input code point is an ident code point or the next two input code points are a valid escape, then: - // Create a . - // If the next 3 input code points would start an ident sequence, set the ’s type flag to "id". - // Consume an ident sequence, and set the ’s value to the returned string. - // Return the . - // Otherwise, return a with its value set to the current input code point. - if (isIdentCodePoint(lookahead[0]) || validEscape(...lookahead)) { - const hashToken = new HashToken(); - if (startsIdentSequence(...lookahead)) { - hashToken.type = 'id'; - } - hashToken.value = consumeIdentSequence(input); - return hashToken; - } else { - return new DelimToken(String.fromCodePoint(codePoint)); - } - } else if (codePoint === 0x0027) { - // U+0027 APOSTROPHE (') - // Consume a string token and return it. - return consumeStringToken(input, codePoint); - } else if (codePoint === 0x0028) { - // U+0028 LEFT PARENTHESIS (() - // Return a <(-token>. - return new LeftParenthesisToken(); - } else if (codePoint === 0x0029) { - // U+0029 RIGHT PARENTHESIS ()) - // Return a <)-token>. - return new RightParenthesisToken(); - } else if (codePoint === 0x002B) { - // U+002B PLUS SIGN (+) - // If the input stream starts with a number, reconsume the current input code point, consume a numeric token, - // and return it. - // Otherwise, return a with its value set to the current input code point. - if (startsNumber(...lookahead)) { - input.reconsume(codePoint); - return consumeNumericToken(input); - } else { - return new DelimToken(String.fromCodePoint(codePoint)); - } - } else if (codePoint === 0x002C) { - // U+002C COMMA (,) - // Return a . - return new CommaToken(); - } else if (codePoint === 0x002D) { - // U+002D HYPHEN-MINUS (-) - if (startsNumber(...input.peek())) { - // If the input stream starts with a number, reconsume the current input code point, consume a numeric token, and return it. - input.reconsume(codePoint); - return consumeNumericToken(input); - } else if (input.peek()[0] === 0x002D && input.peek()[1] === 0x003E) { - // Otherwise, if the next 2 input code points are U+002D HYPHEN-MINUS U+003E GREATER-THAN SIGN (->), consume them and return a . - input.consume(); - input.consume(); - return new CDCToken(); - } else if (startsIdentSequence(...input.peek())) { - // Otherwise, if the input stream starts with an ident sequence, reconsume the current input code point, consume an ident-like token, and return it. - input.reconsume(codePoint); - return consumeIdentLikeToken(input); - } else { - // Otherwise, return a with its value set to the current input code point. - return new DelimToken(String.fromCodePoint(codePoint)); - } - } else if (codePoint === 0x002E) { - // U+002E FULL STOP (.) - if (startsNumber(...input.peek())) { - // If the input stream starts with a number, reconsume the current input code point, consume a numeric token, and return it. - input.reconsume(codePoint); - return consumeNumericToken(input); - } else { - // Otherwise, return a with its value set to the current input code point. - return new DelimToken(String.fromCodePoint(codePoint)); - } - } else if (codePoint === 0x003A) { - // U+003A COLON (:) - // Return a . - return new ColonToken(); - } else if (codePoint === 0x003B) { - // U+003B SEMICOLON (;) - // Return a . - return new SemicolonToken(); - } else if (codePoint === 0x003C) { - // U+003C LESS-THAN SIGN (<) - if (lookahead[0] === 0x0021 && lookahead[1] === 0x002D && lookahead[2] === 0x002D) { - // If the next 3 input code points are U+0021 EXCLAMATION MARK U+002D HYPHEN-MINUS U+002D HYPHEN-MINUS (!--), consume them and return a . - input.consume(); - input.consume(); - input.consume(); - return new CDOToken(); - } else { - // Otherwise, return a with its value set to the current input code point. - return new DelimToken(String.fromCodePoint(codePoint)); - } - } else if (codePoint === 0x0040) { - // U+0040 COMMERCIAL AT (@) - if (startsIdentSequence(...lookahead)) { - // If the next 3 input code points would start an ident sequence, consume an ident sequence, - // create an with its value set to the returned value, and return it. - return new AtKeywordToken(consumeIdentSequence(input)); - } else { - // Otherwise, return a with its value set to the current input code point. - return new DelimToken(String.fromCodePoint(codePoint)); - } - } else if (codePoint === 0x005B) { - // U+005B LEFT SQUARE BRACKET ([) - // Return a <[-token>. - return new LeftSquareBracketToken(); - } else if (codePoint === 0x005C) { - // U+005C REVERSE SOLIDUS (\) - if (validEscape(...lookahead)) { - // If the input stream starts with a valid escape, reconsume the current input code point, consume an ident-like token, and return it. - input.reconsume(codePoint); - return consumeIdentLikeToken(input); - } else { - // Otherwise, this is a parse error. Return a with its value set to the current input code point. - return new DelimToken(String.fromCodePoint(codePoint)); - } - } else if (codePoint === 0x005D) { - // U+005D RIGHT SQUARE BRACKET (]) - // Return a <]-token>. - return new RightSquareBracketToken(); - } else if (codePoint === 0x007B) { - // U+007B LEFT CURLY BRACKET ({) - // Return a <{-token>. - return new LeftCurlyBracketToken(); - } else if (codePoint === 0x007D) { - // U+007D RIGHT CURLY BRACKET (}) - // Return a <}-token>. - return new RightCurlyBracketToken(); - } else if (isDigit(codePoint)) { - // digit - // Reconsume the current input code point, consume a numeric token, and return it. - input.reconsume(codePoint); - return consumeNumericToken(input); - } else if (isIdentStartCodePoint(codePoint)) { - // ident-start code point - // Reconsume the current input code point, consume an ident-like token, and return it. - input.reconsume(codePoint); - return consumeIdentLikeToken(input); - } else if (typeof codePoint === 'undefined') { - // EOF - // Return an . - return undefined; - } else { - // anything else - // Return a with its value set to the current input code point. - return new DelimToken(String.fromCodePoint(codePoint)); - } -} - -/** - * Tokenize a string into an array of CSS tokens. - * @param {string} str - */ -export function tokenizeString(str) { - const input = new InputStream(str); - // To tokenize a stream of code points into a stream of CSS tokens input, repeatedly consume a token from input - // until an is reached, pushing each of the returned tokens into a stream. - const tokens = []; - while (true) { - const token = consumeToken(input); - if (typeof token === 'undefined') { - return tokens; - } else { - tokens.push(token); - } - } -} \ No newline at end of file