![[BACK]](/icons/back.gif){kind=icon}
Return to tokenizer.js CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [LON-CAPA] / loncom / html / adm / LC_math_editor / src|
Return to tokenizer.js CVS log | Up to [LON-CAPA] / loncom / html / adm / LC_math_editor / src |
1.1 damieng 1: /*
2:
3: Copyright (C) 2014 Michigan State University Board of Trustees
4:
5: The JavaScript code in this page is free software: you can
6: redistribute it and/or modify it under the terms of the GNU
7: General Public License (GNU GPL) as published by the Free Software
8: Foundation, either version 3 of the License, or (at your option)
9: any later version. The code is distributed WITHOUT ANY WARRANTY;
10: without even the implied warranty of MERCHANTABILITY or FITNESS
11: FOR A PARTICULAR PURPOSE. See the GNU GPL for more details.
12:
13: As additional permission under GNU GPL version 3 section 7, you
14: may distribute non-source (e.g., minimized or compacted) forms of
15: that code without the copy of the GNU GPL normally required by
16: section 4, provided you include this license notice and a URL
17: through which recipients can access the Corresponding Source.
18:
19: */
20:
1.2 ! damieng 21: "use strict";
! 22:
1.1 damieng 23: /**
24: * String tokenizer. Recognizes only names, numbers, and parser operators.
25: * @constructor
26: * @param {Definitions} defs - Operator definitions
27: * @param {string} text - The text to tokenize
28: */
29: function Tokenizer(defs, text) {
30: this.defs = defs;
31: this.text = text;
32: }
33:
34: /**
35: * Tokenizes the text.
36: * Can throw a ParseException.
37: * @returns {Array.<Token>}
38: */
39: Tokenizer.prototype.tokenize = function() {
40: var c, i, iop, from, tokens, value;
41:
42: i = 0;
43: c = this.text.charAt(i);
44: tokens = [];
45:
46: main:
47: while (c) {
48: from = i;
49:
50: // ignore whitespace
51: if (c <= ' ') {
52: i++;
53: c = this.text.charAt(i);
54: continue;
55: }
56:
57: // check for numbers before operators
58: // (numbers starting with . will not be confused with the . operator)
59: if ((c >= '0' && c <= '9') ||
60: ((c === Definitions.DECIMAL_SIGN_1 || c === Definitions.DECIMAL_SIGN_2) &&
61: (this.text.charAt(i+1) >= '0' && this.text.charAt(i+1) <= '9'))) {
62: value = '';
63:
64: if (c !== Definitions.DECIMAL_SIGN_1 && c !== Definitions.DECIMAL_SIGN_2) {
65: i++;
66: value += c;
67: // Look for more digits.
68: for (;;) {
69: c = this.text.charAt(i);
70: if (c < '0' || c > '9') {
71: break;
72: }
73: i++;
74: value += c;
75: }
76: }
77:
78: // Look for a decimal fraction part.
79: if (c === Definitions.DECIMAL_SIGN_1 || c === Definitions.DECIMAL_SIGN_2) {
80: i++;
81: value += c;
82: for (;;) {
83: c = this.text.charAt(i);
84: if (c < '0' || c > '9') {
85: break;
86: }
87: i += 1;
88: value += c;
89: }
90: }
91:
92: // Look for an exponent part.
93: if (c === 'e' || c === 'E') {
94: i++;
95: value += c;
96: c = this.text.charAt(i);
97: if (c === '-' || c === '+') {
98: i++;
99: value += c;
100: c = this.text.charAt(i);
101: }
102: if (c < '0' || c > '9') {
103: // syntax error in number exponent
104: throw new ParseException("syntax error in number exponent", from, i);
105: }
106: do {
107: i++;
108: value += c;
109: c = this.text.charAt(i);
110: } while (c >= '0' && c <= '9');
111: }
112:
113: /* this is not necessary, as the parser will not recognize the tokens
114: if it is not accepted, and if bad syntax is accepted a * operator will be added
115: // Make sure the next character is not a letter.
116: if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) {
117: // syntax error in number
118: throw new ParseException("syntax error in number", from, i);
119: }
120: */
121:
122: // Convert the string value to a number. If it is finite, then it is a good token.
123: var n = +value.replace(Definitions.DECIMAL_SIGN_1, '.').replace(Definitions.DECIMAL_SIGN_2, '.');
124: if (isFinite(n)) {
125: tokens.push(new Token(Token.NUMBER, from, i - 1, value, null));
126: continue;
127: } else {
128: // syntax error in number
129: throw new ParseException("syntax error in number", from, i);
130: }
131: }
132:
133: // check for operators before names (they could be confused with
134: // variables if they don't use special characters)
135: for (iop = 0; iop < this.defs.operators.length; iop++) {
136: var op = this.defs.operators[iop];
137: if (this.text.substring(i, i+op.id.length) === op.id) {
138: i += op.id.length;
139: c = this.text.charAt(i);
140: tokens.push(new Token(Token.OPERATOR, from, i - 1, op.id, op));
141: continue main;
142: }
143: }
144:
145: // names
1.2 ! damieng 146: if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
! 147: (c >= 'α' && c <= 'ω') || (c >= 'Α' && c <= 'Ω') || c == 'µ') {
1.1 damieng 148: value = c;
149: i++;
150: for (;;) {
151: c = this.text.charAt(i);
152: if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
1.2 ! damieng 153: (c >= 'α' && c <= 'ω') || (c >= 'Α' && c <= 'Ω') || c == 'µ' ||
1.1 damieng 154: (c >= '0' && c <= '9') || c === '_') {
155: value += c;
156: i++;
157: } else {
158: break;
159: }
160: }
161: tokens.push(new Token(Token.NAME, from, i - 1, value, null));
162: continue;
163: }
164:
165: // unrecognized operator
166: throw new ParseException("unrecognized operator", from, i);
167: }
168: return tokens;
169: };