<!-- Tokenised ECMAScript 3, revision 0 standalone
     Created: 2004-05-15T14:43:00UCT+2
     Last revision: 2004-??-??T??:??:??UCT+2

PublicID: "-//liorean.net//DTD TES 3.0//EN"
SystemID: "http://dtd.liorean.net/tes/3.0/"

  (The systemID does not actually exist yet)

Minimal valid document:
[BOF]<!DOCTYPE Program PUBLIC
    "-//liorean.net//DTD TES 3.0//EN"
    "http://dtd.liorean.net/tes/3.0/">
<Program xmlns="http://ns.liorean.net/tes/">
    <EmptyStatement/>
</Program>[EOF]


Explanations of choices and considerations made in writing TES:
The case of TES elements matches the ECMAScript constructs they represent when possible. Thus the element is called FormalParameterList because that is what ECMA-262 3ed calls the corresponding construct. However, for practical reasons TES does not match ECMAScript exactly. The grammar system in DTD has quantifiers, and entities work by pure textual replacement. The grammar system in ECMA-262 3ed works by replacement at need. That means that the recursive way of specifying e.g. a StatementList as either a Statement or a StatementList followed by a Statement is not only impossible in DTD (would lead to infinite recursion), but that there are other ways of achieving the same effect that would be better suited.
In the same way, an XML document were you needed to wrap all other content in a SourceElements just because you add another construct in a global or local situation would lead to absurdly nested documents. XML is much better suited for a simple serial representation, which also benefits the user as it makes documents much easier to write, and also reduces the final document size.
For a number of considerations (such as document size, reduction of document complexity, ease of validation, brevity) I chose to make implicit all static parts of constructs that can be implicit, so that documents only have to contain constructs and their variable parts. The biggest con with this is that it means an unstyled document will not be a valid and executable script and that Microsoft Internet Explorer will require transformations or bindings to display a document as a valid and executable script because of it's bad support for CSS2.
Some ECMAScript constructs, like SourceElement or Statement, are present in the form of internal entities in the TES DTD, but are not included as elements in TES documents. This goes for most constructs that are not recursive or terminal and that does not include variable parts. They are instead represented by one of the constructs that they contain. Thus you e.g. don't have to wrap EmptyStatement in a Statement - you simply use only EmptyStatement.
In ECMAScript, the expression constructs can be seen as a hierarchy ranging from most complex to least complex. For practical purposes, this leads to the requirement to be able to only specify the terminal expression construct, while also requiring that you can use all others as elements where needed. These are present as an internal entity/element combo, where the contents model of the element is exactly that of the internal entity. In an effort to make some ambiguous constructs unambiguous, element content models for statements in chosen situations require the element instead of anything the internal entity may correspond to.

Purpose of TES:
The purpose of TES can be divided in a number of points
1. There currently exists no good way of semantically representing ECMAScript samples in XHTML or other XML based markup languages so that constructs are distinct. TES is made to fill this niche.
2. To increase understanding of code and the inner workings of it, as well as why it works as it does, among JavaScript developers. I believe that a JavaScript developer might understand his own code better, as well as why it works as it does, if he's seen the tokenised result of his code.

    // David "liorean" Andersson, 2004-05-15
-->

<!ENTITY TES.Namespace "http://ns.liorean.net/tes/">

<!--
    ECMAScript construct entities
-->

<!ENTITY % Program.content "%SourceElements;">
<!ENTITY % SourceElements "(%SourceElement;)+"
<!ENTITY % SourceElement "%Statement; | FunctionDeclaration">

<!ENTITY % Statement "(Block | VariableStatement | EmptyStatement | ExpressionStatement | IfStatement | (%IterationStatement;) | ContinueStatement | BreakStatement | ReturnStatement | WithStatement | LabelledStatement | SwitchStatement | ThrowStatement | TryStatement)">

<!ENTITY % Block.content "%StatementList;">
<!ENTITY % StatementList "%Statement;+">

<!ENTITY % VariableStatement.content "%VariableDeclarationList;">
<!ENTITY % VariableDeclarationList "(%VariableDeclaration;)+">
<!ENTITY % VariableDeclarationListNoIn "(%VariableDeclarationNoIn;)+">

<!ENTITY % VarableDeclaration "(Identifier, %Initialiser;)">
<!ENTITY % VarableDeclarationNoIn "(Identifier, %InitialiserNoIn;)">

<!ENTITY % Identifier.content "(#PCDATA)">

<!ENTITY % Initialiser "(Assign, %AssignmentExpression;)?">
<!ENTITY % InitialiserNoIn "(Assign, %AssignmentExpressionNoIn;)?">

<!ENTITY % Expression "(%AssignmentExpression; | Expression)">
<!ENTITY % ExpressionNoIn "(%AssignmentExpressionNoIn; | ExpressionNoIn)">
<!ENTITY % Expression.content "(%AssignmentExpression;)+">
<!ENTITY % ExpressionNoIn.content "(%AssignmentExpressionNoIn;)+">

<!ENTITY % AssignmentExpression "(%ConditionalExpression; | AssignmentExpression)">
<!ENTITY % AssignmentExpressionNoIn "(%ConditionalExpressionNoIn; | AssignmentExpressionNoIn)">
<!ENTITY % AssignmentExpression.content "(%LeftHandSideExpression;, %Operator.assignment;, %AssignmentExpression;)">
<!ENTITY % AssignmentExpressionNoIn.content "(%LeftHandSideExpression;, %Operator.assignment;, %AssignmentExpressionNoIn;)">

<!ENTITY % Operator.assignment "(Assign | MultiplyAssign | DivideAssign | ModuloAssign | AdditionAssign | SubtractionAssign | LeftShiftAssign | SignedRightShiftAssign | UnsignedRightShiftAssign | BitwiseANDAssign | BitwiseXORAssign | BitwiseORAssign)">

<!ENTITY % ConditionalExpression "(%LogicalORExpression; | ConditionalExpression">
<!ENTITY % ConditionalExpressionNoIn "(%LogicalORExpressionNoIn; | ConditionalExpressionNoIn">
<!ENTITY % ConditionalExpression.content "(%LogicalORExpression;, %AssignmentExpression;, %AssignmentExpression;)">
<!ENTITY % ConditionalExpressionNoIn.content "(%LogicalORExpressionNoIn;, %AssignmentExpression;, %AssignmentExpressionNoIn;)">

<!ENTITY % LogicalORExpression "(%LogicalANDExpression; | LogicalORExpression)">
<!ENTITY % LogicalORExpressionNoIn "(%LogicalANDExpressionNoIn; | LogicalORExpressionNoIn)">
<!ENTITY % LogicalORExpression.content "(%LogicalORExpression;, %LogicalANDExpression;">
<!ENTITY % LogicalORExpressionNoIn.content "(%LogicalORExpressionNoIn;, %LogicalANDExpressionNoIn;">

<!ENTITY % LogicalANDExpression "(%BitwiseORExpression; | LogicalANDExpression)">
<!ENTITY % LogicalANDExpressionNoIn "(%BitwiseORExpressionNoIn; | LogicalANDExpressionNoIn)">
<!ENTITY % LogicalANDExpression.content "(%LogicalANDExpression;, %BitwiseORExpression;">
<!ENTITY % LogicalANDExpressionNoIn.content "(%LogicalANDExpressionNoIn;, %BitwiseORExpressionNoIn;">

<!ENTITY % BitwiseORExpression "(%BitwiseXORExpression; | BitwiseORExpression)">
<!ENTITY % BitwiseORExpressionNoIn "(%BitwiseXORExpressionNoIn; | BitwiseORExpressionNoIn)">
<!ENTITY % BitwiseORExpression.content "(%BitwiseORExpression;, %BitwiseXORExpression;)">
<!ENTITY % BitwiseORExpressionNoIn.content "(%BitwiseORExpressionNoIn;, %BitwiseXORExpressionNoIn;)">

<!ENTITY % BitwiseXORExpression "(%BitwiseANDExpression; | BitwiseXORExpression)">
<!ENTITY % BitwiseXORExpressionNoIn "(%BitwiseANDExpressionNoIn; | BitwiseXORExpressionNoIn)">
<!ENTITY % BitwiseXORExpression.content "(%BitwiseXORExpression;, %BitwiseANDExpression;)">
<!ENTITY % BitwiseXORExpressionNoIn.content "(%BitwiseXORExpressionNoIn;, %BitwiseANDExpressionNoIn;)">

<!ENTITY % BitwiseANDExpression "(%EqualityExpression; | BitwiseANDExpression)">
<!ENTITY % BitwiseANDExpressionNoIn "(%EqualityExpressionNoIn; | BitwiseANDExpressionNoIn)">
<!ENTITY % BitwiseANDExpression.content "(%BitwiseANDExpression;, %EqualityExpression;)">
<!ENTITY % BitwiseANDExpressionNoIn.content "(%BitwiseANDExpressionNoIn;, %EqualityExpressionNoIn;)">

<!ENTITY % EqualityExpression "(%RelationalExpression; | EqualityExpression)">
<!ENTITY % EqualityExpressionNoIn "(%RelationalExpressionNoIn; | EqualityExpressionNoIn)">
<!ENTITY % EqualityExpression.content "(%EqualityExpression;, %Operator.equality;, %RelationalExpression;)">
<!ENTITY % EqualityExpressionNoIn.content "(%EqualityExpressionNoIn;, %Operator.equality;, %RelationalExpressionNoIn;)">

<!ENTITY % Operator.equality "(Equals | Does-not-equal | StrictEquals | StrictDoes-not-equal)">

<!ENTITY % RelationalExpression "(%ShiftExpression; | RelationalExpression)">
<!ENTITY % RelationalExpressionNoIn "(%ShiftExpression; | RelationalExpressionNoIn)">
<!ENTITY % RelationalExpression.content "%RelationalExpression;, %Operator.relational;, %ShiftExpression;">
<!ENTITY % RelationalExpressionNoIn.content "%RelationalExpressionNoIn;, %OperatorNoIn.relational;, %ShiftExpression;">

<!ENTITY % Operator.relational "(%OperatorNoIn.relational; | in)">
<!ENTITY % OperatorNoIn.relational "(Less-than | Greater-than | Less-than-or-equal | Greater-than-or-equal | instanceof)">

<!ENTITY % ShiftExpression "(%AdditiveExpression; | ShiftExpression)">
<!ENTITY % ShiftExpression.content "(%ShiftExpression;, %Operator.shift;, %AdditiveExpression;)">

<!ENTITY % Operator.shift "(LeftShift | SignedRightShift | UnsignedRightShift)">

<!ENTITY % AdditiveExpression "(%MultiplicativeExpression; | AdditiveExpression)">
<!ENTITY % AdditiveExpression.content "(%AdditiveExpression;, %Operator.additive;, %MultiplicativeExpression;)">

<!ENTITY % Operator.additive "(Addition | Subtraction)">

<!ENTITY % MultiplicativeExpression "(%UnaryExpression; | MultiplicativeExpression)">
<!ENTITY % MultiplicativeExpression.content "(%MultiplicativeExpression;, %Operator.multiplicative;, %UnaryExpression;)">

<!ENTITY % Operator.multiplicative "(Multiplication | Division | Remainder)">

<!ENTITY % UnaryExpression "(%PostfixExpression; | UnaryExpression)">
<!ENTITY % UnaryExpression.content "(%Operator.unary;, %UnaryExpression;)">

<!ENTITY % Operator.unary "(delete | void | typeof | PrefixIncrement | PrefixDecrement | UnaryPositive | UnaryNegative | BitwiseNOT | LogicalNOT)">

<!ENTITY % PostfixExpression "(%LeftHandSideExpression; | PostfixExpression)">
<!ENTITY % PostfixExpression.content "(%LeftHandSideExpression;, %Operator.postfix;)">

<!ENTITY % Operator.postfix "(PostfixIncrement | PostfixDecrement)">

<!ENTITY % LeftHandSideExpression "(%NewExpression; | %CallExpression;)">

<!-- NewExpression and MemberExpression do not exactly match their ECMAScript constructs here due to recursion limitations in DTD syntax. (More specifically I didn't find a good, practical way to do the distinction between the "NewExpression: new NewExpression" and the "MemberExpression: new MemberExpression Arguments" constructs, so I wrote that possibility into NewExpression.) This does nearly exactly the same thing, though, using DTD friendly syntax. This means a slight element/construct discrepancy between TES 3.0 expressions and ECMAScript 3. If you believe you see a better way to do this that matches the ECMAScript 3 constructs I'd be pleased to hear it. // liorean -->

<!ENTITY % NewExpression "(%MemberExpression; | NewExpression)">
<!ENTITY % NewExpression.content "(%NewExpression;, Arguments)">

<!ENTITY % MemberExpression "(%PrimaryExpression; | %FunctionExpression; | MemberExpression)">
<!ENTITY % MemberExpression.content "(%MemberExpression;, %PropertyAccessor;?)">

<!ENTITY % PropertyAccessor "(Expression | Identifier)">

<!ENTITY % Arguments.content "%ArgumentList;">

<!ENTITY % ArgumentList "(%AssignmentExpression;)*">

<!-- The constructs in PrimaryExpression are for practical reasons not corresponding to the ECMAScript constructs - it would bloat the code too much. Instead, they normally rely on #PCDATA and that the code is intended to typically be compiled from a JavaScript source code file. -->

<!ENTITY % PrimaryExpression "(this | Identifier | %Literal; | ArrayLiteral | ObjectLiteral | Grouping)">

<!ENTITY % Literal "(null | %BooleanLiteral; | %NumericLiteral; | %StringLiteral;)">

<!ENTITY % BooleanLiteral "(true | false)">

<!ENTITY % NumericLiteral "(DecimalLiteral | HexIntegerLiteral)">

<!ENTITY % DecimalLiteral "(#PCDATA)">

<!ENTITY % HexIntegerLiteral "(#PCDATA)">

<!ENTITY % StringLiteral "StringLiteral"> <!-- Separate string types depending, or use only one? I chose a single one here for practical reasons. -->
<!ENTITY % StringLiteral.content "(EscapeSequence | #PCDATA)*">

<!ENTITY % EscapeSequence.content "(#PCDATA)">

<!ENTITY % ArrayLiteral.content "(Elision | %AssignmentExpression;)*">

<!ENTITY % ObjectLiteral.content "%PropertyNameAndValueList;">

<!ENTITY % PropertyNameAndValueList "(%PropertyName;, %AssignmentExpression;)*">

<!ENTITY % PropertyName "(Identifier | StringLiteral | NumericLiteral)">

<!-- Back to the non-primary expressions. -->

<!ENTITY % FunctionExpression "(Identifier? | FormalParameterList | %FunctionBody;)">

<!ENTITY % FormalParameterList.content "(Identifier*)">

<!ENTITY % FunctionBody "%SourceElements;">

<!-- CallExpression deliberately breaks with ECMAScript here and allow NewExpression instead of MemberExpression as it should. Since I force Arguments in NewExpression I can't see where it would introduce any incompatibilities in either direction. ->

<!ENTITY % CallExpression "(CallExpression)">
<!ENTITY % CallExpression.content "((%NewExpression;, Arguments) | (%CallExpression;, Arguments) | (%CallExpression;, %PropertyAccessor;))">

<!ENTITY % IfStatement.content "(%Expression;, %Statement;, %Statement;?)">

<!ENTITY % IterationStatement "Do-WhileStatement | WhileStatement | ForStatement | For-InStatement">

<!ENTITY % Do-WhileStatement.content "(%Statement;, %Expression;)">

<!ENTITY % WhileStatement.content "(%Expression;, %Statement;)">

<!-- Borrowing Elision from the Array syntax to denote optional parameters not specified. -->

<!ENTITY % ForStatement.content "((%ExpressionNoIn; | %VariableDeclarationNoIn; | Elision), (%Expression; | Elision), (%Expression; | Elision), %Statement;)">

<!ENTITY % For-InStatement.content "((%LeftHandSideExpression; | %VariableDeclarationNoIn;), %Statement;)">

<!ENTITY % ContinueStatement.content "(Identifier)?">

<!ENTITY % BreakStatement.content "(Identifier)?">

<!ENTITY % ReturnStatement.content "(%Expression;)?">

<!ENTITY % WithStatement.content "(%Expression;, %Statement;)">

<!ENTITY % SwitchStatement.content "(%Expression;, %CaseBlock;)">

<!ENTITY % CaseBlock "(%CaseClauses;, DefaultClause?, %CaseClauses;)">

<!ENTITY % CaseClauses "(CaseClause)*">

<!ENTITY % CaseClause.content "(%Expression;, %StatementList;?)">

<!ENTITY % DefaultClause.content "(StatementList;?)">

<!ENTITY % LabelledStatement.content "(Identifier, %Statement;)">

<!ENTITY % ThrowStatement.content "(%Expression;)">

<!ENTITY % TryStatement.content "(Block, (Catch | Finally | (Catch, Finally)))">

<!ENTITY % Catch.content "(Identifier, Block)">

<!ENTITY % Finally.content "(Block)">

<!ENTITY % FunctionDeclaration.content "(Identifier, FormalParameterList, %FunctionBody;)">
<!--
    ECMAScript construct elements
-->

<!ELEMENT Program %Program.content;>

<!ELEMENT Block %Block.content;>

<!ELEMENT VariableStatement %VariableStatement.content;>
<!ELEMENT Identifier %Identifier.content;>
<!ELEMENT Assign EMPTY>

<!ELEMENT AssignmentExpression %AssignmentExpression.content;>
<!ELEMENT MultiplyAssign EMPTY>
<!ELEMENT DivideAssign EMPTY>
<!ELEMENT ModuloAssign EMPTY>
<!ELEMENT AdditionAssign EMPTY>
<!ELEMENT SubtractionAssign EMPTY>
<!ELEMENT LeftShiftAssign EMPTY>
<!ELEMENT SignedRightShiftAssign EMPTY>
<!ELEMENT UnsignedRightShiftAssign EMPTY>
<!ELEMENT BitwiseANDAssign EMPTY>
<!ELEMENT BitwiseXORAssign EMPTY>
<!ELEMENT BitwiseORAssign EMPTY>

<!ELEMENT ConditionalExpression %ConditionalExpression.content;>

<!ELEMENT LogicalORExpression %LogicalORExpression.content;>
<!ELEMENT LogicalOR EMPTY>

<!ELEMENT LogicalANDExpression %LogicalANDExpression.content;>
<!ELEMENT LogicalAND EMPTY>

<!ELEMENT BitwiseORExpression %BitwiseORExpression.content;>
<!ELEMENT BitwiseOR EMPTY>

<!ELEMENT BitwiseXORExpression %BitwiseXORExpression.content;>
<!ELEMENT BitwiseXOR EMPTY>

<!ELEMENT BitwiseANDExpression %BitwiseANDExpression.content;>
<!ELEMENT BitwiseAND EMPTY>

<!ELEMENT EqualityExpression %EqualityExpression.content;>
<!ELEMENT Equals EMPTY>
<!ELEMENT Does-not-equal EMPTY>
<!ELEMENT StrictEquals EMPTY>
<!ELEMENT StrictDoes-not-equal EMPTY>

<!ELEMENT RelationalExpression %RelationalExpression.content;>
<!ELEMENT Less-than EMPTY>
<!ELEMENT Greater-than EMPTY>
<!ELEMENT Less-than-or-equal EMPTY>
<!ELEMENT Greater-than-or-equal EMPTY>
<!ELEMENT instanceof EMPTY>
<!ELEMENT in EMPTY>

<!ELEMENT ShiftExpression %ShiftExpression.content;>
<!ELEMENT LeftShift EMPTY>
<!ELEMENT SignedRightShift EMPTY>
<!ELEMENT UnsignedRightShift EMPTY>

<!ELEMENT AdditiveExpression %AdditiveExpression.content;>
<!ELEMENT Addition EMPTY>
<!ELEMENT Subtraction EMPTY>

<!ELEMENT MultiplicativeExpression %MultiplicativeExpression.content;>
<!ELEMENT Multiplication EMPTY>
<!ELEMENT Division EMPTY>
<!ELEMENT Remainder EMPTY>

<!ELEMENT UnaryExpression %UnaryExpression.content;>
<!ELEMENT delete EMPTY>
<!ELEMENT void EMPTY>
<!ELEMENT typeof EMPTY>
<!ELEMENT PrefixIncrement EMPTY>
<!ELEMENT PrefixDecrement EMPTY>
<!ELEMENT UnaryPositive EMPTY>
<!ELEMENT UnaryNegative EMPTY>
<!ELEMENT BitwiseNOT EMPTY>
<!ELEMENT LogicalNOT EMPTY>

<!ELEMENT PostfixExpression %PostfixExpression.content;>
<!ELEMENT PostfixIncrement EMPTY>
<!ELEMENT PostfixDecrement EMPTY>

<!ELEMENT NewExpression %NewExpression.content;>
<!ELEMENT Arguments %Arguments.content>
<!ELEMENT Comma EMPTY>

<!ELEMENT this EMPTY>
<!ELEMENT ArrayLiteral %ArrayLiteral.content;>
<!ELEMENT ObjectLiteral %ObjectLiteral.content;>
<!ELEMENT Grouping %Grouping.content;>

<!ELEMENT null EMPTY>
<!ELEMENT true EMPTY>
<!ELEMENT false EMPTY>
<!ELEMENT DecimalLiteral %DecimalLiteral.content;>
<!ELEMENT HexIntegerLiteral %HexIntegerLiteral.content;>

<!ELEMENT StringLiteral %StringLiteral.content;>
<!ELEMENT EscapeSequence %EscapeSequence.content;>

<!ELEMENT ArrayLiteral %ArrayLiteral.content;>
<!ELEMENT Elision EMPTY>

<!ELEMENT ObjectLiteral %ObjectLiteral.content;>

<!ELEMENT FunctionExpression %FunctionExpression.content;>
<!ELEMENT FormalParameterList %FormalParameterList.content;>

<!ELEMENT MemberExpression %MemberExpression.content;>

<!ELEMENT CallExpression %CallExpression.content;>

<!ELEMENT EmptyStatement EMPTY>

<!ELEMENT ExpressionStatement %Expression;>
<!ELEMENT Expression %Expression.content;>

<!ELEMENT IfStatement %IfStatement.content;>

<!ELEMENT Do-WhileStatement %Do-WhileStatement.content; >
<!ELEMENT WhileStatement %WhileStatement.content;>
<!ELEMENT ForStatement %ForStatement.content;>
<!ELEMENT For-InStatement %For-InStatement.content;>

<!ELEMENT ContinueStatement %ContinueStatement.content;>
<!ELEMENT BreakStatement %BreakStatement.content;>
<!ELEMENT ReturnStatement %ReturnStatement.content;>

<!ELEMENT SwitchStatement %SwitchStatement.content;>
<!ELEMENT CaseClause %CaseClause.content;>
<!ELEMENT DefaultClause %DefaultClause.content;>

<!ELEMENT LabelledStatement %LabelledStatement.content;>

<!ELEMENT ThrowStatement %ThrowStatement.content;>

<!ELEMENT TryStatement %TryStatement.content;>
<!ELEMENT Catch %Finally.content;>

<!ELEMENT FunctionDeclaration %FunctionDeclaration.content;>

<!--
    ECMAScript construct attributes
-->
<!ATTLIST Program
  xmlns         %TES.namespace;          #FIXED
  >
