Annotation of loncom/html/adm/LC_math_editor/src/tokenizer.js, revision 1.1
1.1 ! damieng 1: /*
! 2:
! 3: Copyright (C) 2014 Michigan State University Board of Trustees
! 4:
! 5: The JavaScript code in this page is free software: you can
! 6: redistribute it and/or modify it under the terms of the GNU
! 7: General Public License (GNU GPL) as published by the Free Software
! 8: Foundation, either version 3 of the License, or (at your option)
! 9: any later version. The code is distributed WITHOUT ANY WARRANTY;
! 10: without even the implied warranty of MERCHANTABILITY or FITNESS
! 11: FOR A PARTICULAR PURPOSE. See the GNU GPL for more details.
! 12:
! 13: As additional permission under GNU GPL version 3 section 7, you
! 14: may distribute non-source (e.g., minimized or compacted) forms of
! 15: that code without the copy of the GNU GPL normally required by
! 16: section 4, provided you include this license notice and a URL
! 17: through which recipients can access the Corresponding Source.
! 18:
! 19: */
! 20:
! 21: /**
! 22: * String tokenizer. Recognizes only names, numbers, and parser operators.
! 23: * @constructor
! 24: * @param {Definitions} defs - Operator definitions
! 25: * @param {string} text - The text to tokenize
! 26: */
! 27: function Tokenizer(defs, text) {
! 28: this.defs = defs;
! 29: this.text = text;
! 30: }
! 31:
! 32: /**
! 33: * Tokenizes the text.
! 34: * Can throw a ParseException.
! 35: * @returns {Array.<Token>}
! 36: */
! 37: Tokenizer.prototype.tokenize = function() {
! 38: var c, i, iop, from, tokens, value;
! 39:
! 40: i = 0;
! 41: c = this.text.charAt(i);
! 42: tokens = [];
! 43:
! 44: main:
! 45: while (c) {
! 46: from = i;
! 47:
! 48: // ignore whitespace
! 49: if (c <= ' ') {
! 50: i++;
! 51: c = this.text.charAt(i);
! 52: continue;
! 53: }
! 54:
! 55: // check for numbers before operators
! 56: // (numbers starting with . will not be confused with the . operator)
! 57: if ((c >= '0' && c <= '9') ||
! 58: ((c === Definitions.DECIMAL_SIGN_1 || c === Definitions.DECIMAL_SIGN_2) &&
! 59: (this.text.charAt(i+1) >= '0' && this.text.charAt(i+1) <= '9'))) {
! 60: value = '';
! 61:
! 62: if (c !== Definitions.DECIMAL_SIGN_1 && c !== Definitions.DECIMAL_SIGN_2) {
! 63: i++;
! 64: value += c;
! 65: // Look for more digits.
! 66: for (;;) {
! 67: c = this.text.charAt(i);
! 68: if (c < '0' || c > '9') {
! 69: break;
! 70: }
! 71: i++;
! 72: value += c;
! 73: }
! 74: }
! 75:
! 76: // Look for a decimal fraction part.
! 77: if (c === Definitions.DECIMAL_SIGN_1 || c === Definitions.DECIMAL_SIGN_2) {
! 78: i++;
! 79: value += c;
! 80: for (;;) {
! 81: c = this.text.charAt(i);
! 82: if (c < '0' || c > '9') {
! 83: break;
! 84: }
! 85: i += 1;
! 86: value += c;
! 87: }
! 88: }
! 89:
! 90: // Look for an exponent part.
! 91: if (c === 'e' || c === 'E') {
! 92: i++;
! 93: value += c;
! 94: c = this.text.charAt(i);
! 95: if (c === '-' || c === '+') {
! 96: i++;
! 97: value += c;
! 98: c = this.text.charAt(i);
! 99: }
! 100: if (c < '0' || c > '9') {
! 101: // syntax error in number exponent
! 102: throw new ParseException("syntax error in number exponent", from, i);
! 103: }
! 104: do {
! 105: i++;
! 106: value += c;
! 107: c = this.text.charAt(i);
! 108: } while (c >= '0' && c <= '9');
! 109: }
! 110:
! 111: /* this is not necessary, as the parser will not recognize the tokens
! 112: if it is not accepted, and if bad syntax is accepted a * operator will be added
! 113: // Make sure the next character is not a letter.
! 114: if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) {
! 115: // syntax error in number
! 116: throw new ParseException("syntax error in number", from, i);
! 117: }
! 118: */
! 119:
! 120: // Convert the string value to a number. If it is finite, then it is a good token.
! 121: var n = +value.replace(Definitions.DECIMAL_SIGN_1, '.').replace(Definitions.DECIMAL_SIGN_2, '.');
! 122: if (isFinite(n)) {
! 123: tokens.push(new Token(Token.NUMBER, from, i - 1, value, null));
! 124: continue;
! 125: } else {
! 126: // syntax error in number
! 127: throw new ParseException("syntax error in number", from, i);
! 128: }
! 129: }
! 130:
! 131: // check for operators before names (they could be confused with
! 132: // variables if they don't use special characters)
! 133: for (iop = 0; iop < this.defs.operators.length; iop++) {
! 134: var op = this.defs.operators[iop];
! 135: if (this.text.substring(i, i+op.id.length) === op.id) {
! 136: i += op.id.length;
! 137: c = this.text.charAt(i);
! 138: tokens.push(new Token(Token.OPERATOR, from, i - 1, op.id, op));
! 139: continue main;
! 140: }
! 141: }
! 142:
! 143: // names
! 144: if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) {
! 145: value = c;
! 146: i++;
! 147: for (;;) {
! 148: c = this.text.charAt(i);
! 149: if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
! 150: (c >= '0' && c <= '9') || c === '_') {
! 151: value += c;
! 152: i++;
! 153: } else {
! 154: break;
! 155: }
! 156: }
! 157: tokens.push(new Token(Token.NAME, from, i - 1, value, null));
! 158: continue;
! 159: }
! 160:
! 161: // unrecognized operator
! 162: throw new ParseException("unrecognized operator", from, i);
! 163: }
! 164: return tokens;
! 165: };
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>