openshot-audio  0.1.4
juce_CPlusPlusCodeTokeniserFunctions.h
Go to the documentation of this file.
1 /*
2  ==============================================================================
3 
4  This file is part of the JUCE library.
5  Copyright (c) 2015 - ROLI Ltd.
6 
7  Permission is granted to use this software under the terms of either:
8  a) the GPL v2 (or any later version)
9  b) the Affero GPL v3
10 
11  Details of these licenses can be found at: www.gnu.org/licenses
12 
13  JUCE is distributed in the hope that it will be useful, but WITHOUT ANY
14  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
15  A PARTICULAR PURPOSE. See the GNU General Public License for more details.
16 
17  ------------------------------------------------------------------------------
18 
19  To release a closed-source product which uses JUCE, commercial licenses are
20  available: visit www.juce.com for more information.
21 
22  ==============================================================================
23 */
24 
25 #ifndef JUCE_CPLUSPLUSCODETOKENISERFUNCTIONS_H_INCLUDED
26 #define JUCE_CPLUSPLUSCODETOKENISERFUNCTIONS_H_INCLUDED
27 
28 
29 //==============================================================================
33 {
34  static bool isIdentifierStart (const juce_wchar c) noexcept
35  {
37  || c == '_' || c == '@';
38  }
39 
40  static bool isIdentifierBody (const juce_wchar c) noexcept
41  {
43  || c == '_' || c == '@';
44  }
45 
46  static bool isReservedKeyword (String::CharPointerType token, const int tokenLength) noexcept
47  {
48  static const char* const keywords2Char[] =
49  { "if", "do", "or", nullptr };
50 
51  static const char* const keywords3Char[] =
52  { "for", "int", "new", "try", "xor", "and", "asm", "not", nullptr };
53 
54  static const char* const keywords4Char[] =
55  { "bool", "void", "this", "true", "long", "else", "char",
56  "enum", "case", "goto", "auto", nullptr };
57 
58  static const char* const keywords5Char[] =
59  { "float", "const", "while", "break", "false", "catch", "class", "bitor",
60  "compl", "or_eq", "short", "throw", "union", "using", "final", nullptr };
61 
62  static const char* const keywords6Char[] =
63  { "return", "and_eq", "bitand", "delete", "double", "export", "extern",
64  "friend", "inline", "not_eq", "public", "signed", "sizeof", "static",
65  "struct", "switch", "typeid", "xor_eq", nullptr };
66 
67  static const char* const keywords7Char[] =
68  { "nullptr", "alignas", "alignof", "default", "mutable", "private",
69  "typedef", "virtual", "wchar_t", nullptr };
70 
71  static const char* const keywordsOther[] =
72  { "char16_t", "char32_t", "const_cast", "constexpr", "continue", "decltype", "dynamic_cast",
73  "explicit", "namespace", "noexcept", "operator", "protected", "register", "reinterpret_cast",
74  "static_assert", "static_cast", "template", "thread_local", "typename", "unsigned", "volatile",
75  "@class", "@dynamic", "@end", "@implementation", "@interface", "@public", "@private",
76  "@protected", "@property", "@synthesize", nullptr };
77 
78  const char* const* k;
79 
80  switch (tokenLength)
81  {
82  case 2: k = keywords2Char; break;
83  case 3: k = keywords3Char; break;
84  case 4: k = keywords4Char; break;
85  case 5: k = keywords5Char; break;
86  case 6: k = keywords6Char; break;
87  case 7: k = keywords7Char; break;
88 
89  default:
90  if (tokenLength < 2 || tokenLength > 16)
91  return false;
92 
93  k = keywordsOther;
94  break;
95  }
96 
97  for (int i = 0; k[i] != 0; ++i)
98  if (token.compare (CharPointer_ASCII (k[i])) == 0)
99  return true;
100 
101  return false;
102  }
103 
104  template <typename Iterator>
105  static int parseIdentifier (Iterator& source) noexcept
106  {
107  int tokenLength = 0;
108  String::CharPointerType::CharType possibleIdentifier [100];
109  String::CharPointerType possible (possibleIdentifier);
110 
111  while (isIdentifierBody (source.peekNextChar()))
112  {
113  const juce_wchar c = source.nextChar();
114 
115  if (tokenLength < 20)
116  possible.write (c);
117 
118  ++tokenLength;
119  }
120 
121  if (tokenLength > 1 && tokenLength <= 16)
122  {
123  possible.writeNull();
124 
125  if (isReservedKeyword (String::CharPointerType (possibleIdentifier), tokenLength))
127  }
128 
130  }
131 
132  template <typename Iterator>
133  static bool skipNumberSuffix (Iterator& source)
134  {
135  const juce_wchar c = source.peekNextChar();
136  if (c == 'l' || c == 'L' || c == 'u' || c == 'U')
137  source.skip();
138 
139  if (CharacterFunctions::isLetterOrDigit (source.peekNextChar()))
140  return false;
141 
142  return true;
143  }
144 
145  static bool isHexDigit (const juce_wchar c) noexcept
146  {
147  return (c >= '0' && c <= '9')
148  || (c >= 'a' && c <= 'f')
149  || (c >= 'A' && c <= 'F');
150  }
151 
152  template <typename Iterator>
153  static bool parseHexLiteral (Iterator& source) noexcept
154  {
155  if (source.peekNextChar() == '-')
156  source.skip();
157 
158  if (source.nextChar() != '0')
159  return false;
160 
161  juce_wchar c = source.nextChar();
162  if (c != 'x' && c != 'X')
163  return false;
164 
165  int numDigits = 0;
166  while (isHexDigit (source.peekNextChar()))
167  {
168  ++numDigits;
169  source.skip();
170  }
171 
172  if (numDigits == 0)
173  return false;
174 
175  return skipNumberSuffix (source);
176  }
177 
178  static bool isOctalDigit (const juce_wchar c) noexcept
179  {
180  return c >= '0' && c <= '7';
181  }
182 
183  template <typename Iterator>
184  static bool parseOctalLiteral (Iterator& source) noexcept
185  {
186  if (source.peekNextChar() == '-')
187  source.skip();
188 
189  if (source.nextChar() != '0')
190  return false;
191 
192  if (! isOctalDigit (source.nextChar()))
193  return false;
194 
195  while (isOctalDigit (source.peekNextChar()))
196  source.skip();
197 
198  return skipNumberSuffix (source);
199  }
200 
201  static bool isDecimalDigit (const juce_wchar c) noexcept
202  {
203  return c >= '0' && c <= '9';
204  }
205 
206  template <typename Iterator>
207  static bool parseDecimalLiteral (Iterator& source) noexcept
208  {
209  if (source.peekNextChar() == '-')
210  source.skip();
211 
212  int numChars = 0;
213  while (isDecimalDigit (source.peekNextChar()))
214  {
215  ++numChars;
216  source.skip();
217  }
218 
219  if (numChars == 0)
220  return false;
221 
222  return skipNumberSuffix (source);
223  }
224 
225  template <typename Iterator>
226  static bool parseFloatLiteral (Iterator& source) noexcept
227  {
228  if (source.peekNextChar() == '-')
229  source.skip();
230 
231  int numDigits = 0;
232 
233  while (isDecimalDigit (source.peekNextChar()))
234  {
235  source.skip();
236  ++numDigits;
237  }
238 
239  const bool hasPoint = (source.peekNextChar() == '.');
240 
241  if (hasPoint)
242  {
243  source.skip();
244 
245  while (isDecimalDigit (source.peekNextChar()))
246  {
247  source.skip();
248  ++numDigits;
249  }
250  }
251 
252  if (numDigits == 0)
253  return false;
254 
255  juce_wchar c = source.peekNextChar();
256  const bool hasExponent = (c == 'e' || c == 'E');
257 
258  if (hasExponent)
259  {
260  source.skip();
261 
262  c = source.peekNextChar();
263  if (c == '+' || c == '-')
264  source.skip();
265 
266  int numExpDigits = 0;
267  while (isDecimalDigit (source.peekNextChar()))
268  {
269  source.skip();
270  ++numExpDigits;
271  }
272 
273  if (numExpDigits == 0)
274  return false;
275  }
276 
277  c = source.peekNextChar();
278  if (c == 'f' || c == 'F')
279  source.skip();
280  else if (! (hasExponent || hasPoint))
281  return false;
282 
283  return true;
284  }
285 
286  template <typename Iterator>
287  static int parseNumber (Iterator& source)
288  {
289  const Iterator original (source);
290 
292  source = original;
293 
295  source = original;
296 
298  source = original;
299 
301  source = original;
302 
304  }
305 
306  template <typename Iterator>
307  static void skipQuotedString (Iterator& source) noexcept
308  {
309  const juce_wchar quote = source.nextChar();
310 
311  for (;;)
312  {
313  const juce_wchar c = source.nextChar();
314 
315  if (c == quote || c == 0)
316  break;
317 
318  if (c == '\\')
319  source.skip();
320  }
321  }
322 
323  template <typename Iterator>
324  static void skipComment (Iterator& source) noexcept
325  {
326  bool lastWasStar = false;
327 
328  for (;;)
329  {
330  const juce_wchar c = source.nextChar();
331 
332  if (c == 0 || (c == '/' && lastWasStar))
333  break;
334 
335  lastWasStar = (c == '*');
336  }
337  }
338 
339  template <typename Iterator>
340  static void skipPreprocessorLine (Iterator& source) noexcept
341  {
342  bool lastWasBackslash = false;
343 
344  for (;;)
345  {
346  const juce_wchar c = source.peekNextChar();
347 
348  if (c == '"')
349  {
350  skipQuotedString (source);
351  continue;
352  }
353 
354  if (c == '/')
355  {
356  Iterator next (source);
357  next.skip();
358  const juce_wchar c2 = next.peekNextChar();
359 
360  if (c2 == '/' || c2 == '*')
361  return;
362  }
363 
364  if (c == 0)
365  break;
366 
367  if (c == '\n' || c == '\r')
368  {
369  source.skipToEndOfLine();
370 
371  if (lastWasBackslash)
372  skipPreprocessorLine (source);
373 
374  break;
375  }
376 
377  lastWasBackslash = (c == '\\');
378  source.skip();
379  }
380  }
381 
382  template <typename Iterator>
383  static void skipIfNextCharMatches (Iterator& source, const juce_wchar c) noexcept
384  {
385  if (source.peekNextChar() == c)
386  source.skip();
387  }
388 
389  template <typename Iterator>
390  static void skipIfNextCharMatches (Iterator& source, const juce_wchar c1, const juce_wchar c2) noexcept
391  {
392  const juce_wchar c = source.peekNextChar();
393 
394  if (c == c1 || c == c2)
395  source.skip();
396  }
397 
398  template <typename Iterator>
399  static int readNextToken (Iterator& source)
400  {
401  source.skipWhitespace();
402 
403  const juce_wchar firstChar = source.peekNextChar();
404 
405  switch (firstChar)
406  {
407  case 0:
408  break;
409 
410  case '0': case '1': case '2': case '3': case '4':
411  case '5': case '6': case '7': case '8': case '9':
412  case '.':
413  {
414  int result = parseNumber (source);
415 
417  {
418  source.skip();
419 
420  if (firstChar == '.')
422  }
423 
424  return result;
425  }
426 
427  case ',':
428  case ';':
429  case ':':
430  source.skip();
432 
433  case '(': case ')':
434  case '{': case '}':
435  case '[': case ']':
436  source.skip();
438 
439  case '"':
440  case '\'':
441  skipQuotedString (source);
443 
444  case '+':
445  source.skip();
446  skipIfNextCharMatches (source, '+', '=');
448 
449  case '-':
450  {
451  source.skip();
452  int result = parseNumber (source);
453 
455  {
456  skipIfNextCharMatches (source, '-', '=');
458  }
459 
460  return result;
461  }
462 
463  case '*': case '%':
464  case '=': case '!':
465  source.skip();
466  skipIfNextCharMatches (source, '=');
468 
469  case '/':
470  {
471  source.skip();
472  juce_wchar nextChar = source.peekNextChar();
473 
474  if (nextChar == '/')
475  {
476  source.skipToEndOfLine();
478  }
479 
480  if (nextChar == '*')
481  {
482  source.skip();
483  skipComment (source);
485  }
486 
487  if (nextChar == '=')
488  source.skip();
489 
491  }
492 
493  case '?':
494  case '~':
495  source.skip();
497 
498  case '<': case '>':
499  case '|': case '&': case '^':
500  source.skip();
501  skipIfNextCharMatches (source, firstChar);
502  skipIfNextCharMatches (source, '=');
504 
505  case '#':
506  skipPreprocessorLine (source);
508 
509  default:
510  if (isIdentifierStart (firstChar))
511  return parseIdentifier (source);
512 
513  source.skip();
514  break;
515  }
516 
518  }
519 
524  {
525  StringIterator (const String& s) noexcept : t (s.getCharPointer()), numChars (0) {}
526  StringIterator (String::CharPointerType s) noexcept : t (s), numChars (0) {}
527 
528  juce_wchar nextChar() noexcept { if (isEOF()) return 0; ++numChars; return t.getAndAdvance(); }
530  void skip() noexcept { if (! isEOF()) { ++t; ++numChars; } }
531  void skipWhitespace() noexcept { while (t.isWhitespace()) skip(); }
532  void skipToEndOfLine() noexcept { while (*t != '\r' && *t != '\n' && *t != 0) skip(); }
533  bool isEOF() const noexcept { return t.isEmpty(); }
534 
535  String::CharPointerType t;
536  int numChars;
537  };
538 
539  //==============================================================================
550  static void writeEscapeChars (OutputStream& out, const char* utf8, const int numBytesToRead,
551  const int maxCharsOnLine, const bool breakAtNewLines,
552  const bool replaceSingleQuotes, const bool allowStringBreaks)
553  {
554  int charsOnLine = 0;
555  bool lastWasHexEscapeCode = false;
556 
557  for (int i = 0; i < numBytesToRead || numBytesToRead < 0; ++i)
558  {
559  const unsigned char c = (unsigned char) utf8[i];
560  bool startNewLine = false;
561 
562  switch (c)
563  {
564  case '\t': out << "\\t"; lastWasHexEscapeCode = false; charsOnLine += 2; break;
565  case '\r': out << "\\r"; lastWasHexEscapeCode = false; charsOnLine += 2; break;
566  case '\n': out << "\\n"; lastWasHexEscapeCode = false; charsOnLine += 2; startNewLine = breakAtNewLines; break;
567  case '\\': out << "\\\\"; lastWasHexEscapeCode = false; charsOnLine += 2; break;
568  case '\"': out << "\\\""; lastWasHexEscapeCode = false; charsOnLine += 2; break;
569 
570  case 0:
571  if (numBytesToRead < 0)
572  return;
573 
574  out << "\\0";
575  lastWasHexEscapeCode = true;
576  charsOnLine += 2;
577  break;
578 
579  case '\'':
580  if (replaceSingleQuotes)
581  {
582  out << "\\\'";
583  lastWasHexEscapeCode = false;
584  charsOnLine += 2;
585  break;
586  }
587 
588  // deliberate fall-through...
589 
590  default:
591  if (c >= 32 && c < 127 && ! (lastWasHexEscapeCode // (have to avoid following a hex escape sequence with a valid hex digit)
593  {
594  out << (char) c;
595  lastWasHexEscapeCode = false;
596  ++charsOnLine;
597  }
598  else if (allowStringBreaks && lastWasHexEscapeCode && c >= 32 && c < 127)
599  {
600  out << "\"\"" << (char) c;
601  lastWasHexEscapeCode = false;
602  charsOnLine += 3;
603  }
604  else
605  {
606  out << (c < 16 ? "\\x0" : "\\x") << String::toHexString ((int) c);
607  lastWasHexEscapeCode = true;
608  charsOnLine += 4;
609  }
610 
611  break;
612  }
613 
614  if ((startNewLine || (maxCharsOnLine > 0 && charsOnLine >= maxCharsOnLine))
615  && (numBytesToRead < 0 || i < numBytesToRead - 1))
616  {
617  charsOnLine = 0;
618  out << "\"" << newLine << "\"";
619  lastWasHexEscapeCode = false;
620  }
621  }
622  }
623 
632  static String addEscapeChars (const String& s)
633  {
635  writeEscapeChars (mo, s.toRawUTF8(), -1, -1, false, true, true);
636  return mo.toString();
637  }
638 };
639 
640 
641 #endif // JUCE_CPLUSPLUSCODETOKENISERFUNCTIONS_H_INCLUDED
Definition: juce_CPlusPlusCodeTokeniser.h:55
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:523
Definition: juce_CPlusPlusCodeTokeniser.h:61
static bool isHexDigit(const juce_wchar c) noexcept
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:145
static String toHexString(int number)
Definition: juce_String.cpp:1925
static void writeEscapeChars(OutputStream &out, const char *utf8, const int numBytesToRead, const int maxCharsOnLine, const bool breakAtNewLines, const bool replaceSingleQuotes, const bool allowStringBreaks)
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:550
String::CharPointerType t
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:535
static void skipPreprocessorLine(Iterator &source) noexcept
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:340
static bool parseOctalLiteral(Iterator &source) noexcept
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:184
juce_wchar nextChar() noexcept
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:528
#define noexcept
Definition: juce_CompilerSupport.h:141
static bool parseHexLiteral(Iterator &source) noexcept
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:153
Definition: juce_CPlusPlusCodeTokeniser.h:60
Definition: juce_CPlusPlusCodeTokeniser.h:54
static bool isOctalDigit(const juce_wchar c) noexcept
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:178
static bool isReservedKeyword(String::CharPointerType token, const int tokenLength) noexcept
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:46
void skipToEndOfLine() noexcept
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:532
NewLine newLine
Definition: juce_core.cpp:35
static int getHexDigitValue(juce_wchar digit) noexcept
Definition: juce_CharacterFunctions.cpp:111
static bool isLetterOrDigit(char character) noexcept
Definition: juce_CharacterFunctions.cpp:99
Definition: juce_String.h:43
static void skipComment(Iterator &source) noexcept
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:324
static void skipQuotedString(Iterator &source) noexcept
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:307
static bool isLetter(char character) noexcept
Definition: juce_CharacterFunctions.cpp:88
Definition: juce_CPlusPlusCodeTokeniser.h:59
String toString() const
Definition: juce_MemoryOutputStream.cpp:201
static int parseIdentifier(Iterator &source) noexcept
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:105
void skip() noexcept
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:530
static bool skipNumberSuffix(Iterator &source)
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:133
int numChars
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:536
static bool isIdentifierBody(const juce_wchar c) noexcept
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:40
static String addEscapeChars(const String &s)
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:632
Definition: juce_CPlusPlusCodeTokeniser.h:57
bool isEOF() const noexcept
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:533
static bool parseDecimalLiteral(Iterator &source) noexcept
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:207
Definition: juce_CPlusPlusCodeTokeniser.h:58
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:32
Definition: juce_OutputStream.h:42
StringIterator(const String &s) noexcept
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:525
Definition: juce_CharPointer_ASCII.h:42
Definition: juce_CPlusPlusCodeTokeniser.h:56
static bool parseFloatLiteral(Iterator &source) noexcept
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:226
static bool isDecimalDigit(const juce_wchar c) noexcept
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:201
juce_wchar peekNextChar() noexcept
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:529
static int readNextToken(Iterator &source)
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:399
static void skipIfNextCharMatches(Iterator &source, const juce_wchar c) noexcept
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:383
static int parseNumber(Iterator &source)
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:287
Definition: juce_MemoryOutputStream.h:40
static bool isIdentifierStart(const juce_wchar c) noexcept
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:34
const char * toRawUTF8() const
Definition: juce_String.cpp:2061
Definition: juce_CPlusPlusCodeTokeniser.h:52
Definition: juce_CPlusPlusCodeTokeniser.h:53
void skipWhitespace() noexcept
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:531
wchar_t juce_wchar
Definition: juce_CharacterFunctions.h:49
StringIterator(String::CharPointerType s) noexcept
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:526
Definition: juce_CPlusPlusCodeTokeniser.h:62
static void skipIfNextCharMatches(Iterator &source, const juce_wchar c1, const juce_wchar c2) noexcept
Definition: juce_CPlusPlusCodeTokeniserFunctions.h:390