change folder structure and control files to newest concept

2026-04-28 13:01:38 +02:00 · 2018-09-20 16:38:35 +02:00
parent 224fd33794
commit e25c30f942
2747 changed files with 21 additions and 80 deletions
@@ -0,0 +1,130 @@
+package antlr;
+
+/* ANTLR Translator Generator
+ * Project led by Terence Parr at http://www.jGuru.com
+ * Software rights: http://www.antlr.org/RIGHTS.html
+ *
+ * $Id$
+ */
+
+import java.io.*;
+
+import antlr.collections.*;
+
+
+/** A CommonAST whose initialization copies hidden token
+ *  information from the Token used to create a node.
+ */
+public class ExtendedCommonASTWithHiddenTokens 
+    extends CommonASTWithHiddenTokens {
+
+    public ExtendedCommonASTWithHiddenTokens() {
+        super();
+    }
+
+    public ExtendedCommonASTWithHiddenTokens(Token tok) {
+        super(tok);
+    }
+
+    public void initialize(AST ast) {
+        ExtendedCommonASTWithHiddenTokens a = 
+	    (ExtendedCommonASTWithHiddenTokens)ast;
+        super.initialize(a);
+        hiddenBefore = a.getHiddenBefore();
+        hiddenAfter = a.getHiddenAfter();
+    }
+
+    public String getHiddenAfterString() {
+
+	CommonHiddenStreamToken t;
+	StringBuilder hiddenAfterString = new StringBuilder(100);
+
+	for ( t = hiddenAfter ; t != null ; t = t.getHiddenAfter() ) {
+	    hiddenAfterString.append(t.getText());
+	}
+
+	return hiddenAfterString.toString();
+    }
+
+    public String getHiddenBeforeString() {
+
+	antlr.CommonHiddenStreamToken 
+	    child = null,
+	    parent = hiddenBefore;
+
+	// if there aren't any hidden tokens here, quietly return
+	// 
+	if (parent == null) {
+	    return "";
+	}
+
+	// traverse back to the head of the list of tokens before this node
+	do {
+	    child = parent;
+	    parent = child.getHiddenBefore();
+	} while (parent != null);
+
+	// dump that list
+
+	StringBuilder hiddenBeforeString = new StringBuilder(100);
+
+	for ( CommonHiddenStreamToken t = child; t != null ; 
+	      t = t.getHiddenAfter() ) {
+	    hiddenBeforeString.append(t.getText());
+	}
+
+	return hiddenBeforeString.toString();
+    }
+
+    public void xmlSerializeNode(Writer out) throws IOException {
+        StringBuilder sb = new StringBuilder(100);
+        sb.append("<");
+        sb.append(getClass().getName() + " ");
+
+        sb.append("hiddenBeforeString=\"" +
+            encode(getHiddenBeforeString()) +
+            "\" text=\"" + encode(getText()) + "\" type=\"" +
+            getType() + "\" hiddenAfterString=\"" +
+            encode(getHiddenAfterString()) + "\"/>");
+        out.write(sb.toString());
+    }
+
+    public void xmlSerializeRootOpen(Writer out) throws IOException {
+        StringBuilder sb = new StringBuilder(100);
+        sb.append("<");
+        sb.append(getClass().getName() + " ");
+        sb.append("hiddenBeforeString=\"" +
+            encode(getHiddenBeforeString()) +
+            "\" text=\"" + encode(getText()) + "\" type=\"" +
+            getType() + "\" hiddenAfterString=\"" +
+            encode(getHiddenAfterString()) + "\">\n");
+        out.write(sb.toString());
+    }
+
+    public void xmlSerializeRootClose(Writer out)
+        throws IOException {
+        out.write("</" + getClass().getName() + ">\n");
+    }
+
+    public void xmlSerialize(Writer out) throws IOException {
+        // print out this node and all siblings
+        for (AST node = this;
+             node != null;
+             node = node.getNextSibling()) {
+            if (node.getFirstChild() == null) {
+                // print guts (class name, attributes)
+                ((BaseAST)node).xmlSerializeNode(out);
+            }
+            else {
+                ((BaseAST)node).xmlSerializeRootOpen(out);
+
+                // print children
+                ((BaseAST)node.getFirstChild()).xmlSerialize(out);
+
+                // print end tag
+                ((BaseAST)node).xmlSerializeRootClose(out);
+            }
+        }
+    }
+
+}
@@ -0,0 +1,219 @@
+/* -*- mode: java; c-basic-offset: 2; indent-tabs-mode: nil -*- */
+
+package antlr;
+
+
+import antlr.collections.impl.BitSet;
+
+/**
+ * This class provides TokenStreamHiddenTokenFilters with the concept of
+ * tokens which can be copied so that they are seen by both the hidden token
+ * stream as well as the parser itself.  This is useful when one wants to use
+ * an existing parser (like the Java parser included with ANTLR) that throws
+ * away some tokens to create a parse tree which can be used to spit out
+ * a copy of the code with only minor modifications.
+ *
+ * This code is partially derived from the public domain ANLTR TokenStream
+ */
+public class TokenStreamCopyingHiddenTokenFilter
+  extends TokenStreamHiddenTokenFilter
+  implements TokenStream {
+
+  protected BitSet copyMask;
+  CommonHiddenStreamToken hiddenCopy = null;
+
+  public TokenStreamCopyingHiddenTokenFilter(TokenStream input) {
+    super(input);
+    copyMask = new BitSet();
+  }
+
+  /**
+   * Indicate that all tokens of type tokenType should be copied.  The copy
+   * is put in the stream of hidden tokens, and the original is returned in the
+   * stream of normal tokens.
+   *
+   * @param tokenType   integer representing the token type to copied
+   */
+  public void copy(int tokenType) {
+    copyMask.add(tokenType);
+  }
+
+  /**
+   * Create a clone of the important parts of the given token.  Note that this
+   * does NOT copy the hiddenBefore and hiddenAfter fields.
+   *
+   * @param t   token to partially clone
+   * @return    newly created partial clone
+   */
+  public CommonHiddenStreamToken partialCloneToken(CommonHiddenStreamToken t) {
+
+    CommonHiddenStreamToken u = new CommonHiddenStreamToken(t.getType(),
+                                                            t.getText());
+    u.setColumn(t.getColumn());
+    u.setLine(t.getLine());
+    u.setFilename(t.getFilename());
+
+    return u;
+  }
+
+  public void linkAndCopyToken(CommonHiddenStreamToken prev,
+                               CommonHiddenStreamToken monitored) {
+    // create a copy of the token in the lookahead for use as hidden token
+    hiddenCopy = partialCloneToken(LA(1));
+
+    // attach copy to the previous token, whether hidden or monitored
+    prev.setHiddenAfter(hiddenCopy);
+
+    // if previous token was hidden, set the hiddenBefore pointer of the
+    // copy to point back to it
+    if (prev != monitored) {
+      hiddenCopy.setHiddenBefore(prev);
+    }
+
+    // we don't want the non-hidden copy to link back to the hidden
+    // copy on the next pass through this function, so we leave
+    // lastHiddenToken alone
+
+    //System.err.println("hidden copy: " + hiddenCopy.toString());
+
+    return;
+  }
+
+  private void consumeFirst() throws TokenStreamException {
+    consume(); // get first token of input stream
+
+    // Handle situation where hidden or discarded tokens
+    // appear first in input stream
+    CommonHiddenStreamToken p=null;
+
+    // while hidden, copied, or discarded scarf tokens
+    while ( hideMask.member(LA(1).getType()) ||
+            discardMask.member(LA(1).getType()) ||
+            copyMask.member(LA(1).getType()) ) {
+
+      // if we've hit one of the tokens that needs to be copied, we copy it
+      // and then break out of the loop, because the parser needs to see it
+      // too
+      //
+      if (copyMask.member(LA(1).getType())) {
+
+        // copy the token in the lookahead
+        hiddenCopy = partialCloneToken(LA(1));
+
+        // if there's an existing token before this, link that and the
+        // copy together
+        if (p != null) {
+          p.setHiddenAfter(hiddenCopy);
+          hiddenCopy.setHiddenBefore(p); // double-link
+        }
+
+        lastHiddenToken = hiddenCopy;
+        if (firstHidden == null) {
+          firstHidden = hiddenCopy;
+        }
+
+        // we don't want to consume this token, because it also needs to
+        // be passed through to the parser, so break out of the while look
+        // entirely
+        //
+        break;
+      } else if (hideMask.member(LA(1).getType())) {
+        if (p != null) {
+          p.setHiddenAfter(LA(1));
+          LA(1).setHiddenBefore(p); // double-link
+        }
+        p = LA(1);
+
+        lastHiddenToken = p;
+        if (firstHidden == null) {
+          firstHidden = p; // record hidden token if first
+        }
+      }
+      consume();
+    }
+  }
+
+  /** Return the next monitored token.
+   *  Test the token following the monitored token.
+   *  If following is another monitored token, save it
+   *  for the next invocation of nextToken (like a single
+   *  lookahead token) and return it then.
+   *  If following is unmonitored, nondiscarded (hidden)
+   *  channel token, add it to the monitored token.
+   *
+   *  Note: EOF must be a monitored Token.
+   */
+  public Token nextToken() throws TokenStreamException {
+    // handle an initial condition; don't want to get lookahead
+    // token of this splitter until first call to nextToken
+    if (LA(1) == null) {
+      consumeFirst();
+    }
+
+    //System.err.println();
+
+    // we always consume hidden tokens after monitored, thus,
+    // upon entry LA(1) is a monitored token.
+    CommonHiddenStreamToken monitored = LA(1);
+
+    // point to hidden tokens found during last invocation
+    monitored.setHiddenBefore(lastHiddenToken);
+    lastHiddenToken = null;
+
+    // Look for hidden tokens, hook them into list emanating
+    // from the monitored tokens.
+    consume();
+    CommonHiddenStreamToken prev = monitored;
+
+    // deal with as many not-purely-monitored tokens as possible
+    while ( hideMask.member(LA(1).getType()) ||
+            discardMask.member(LA(1).getType()) ||
+            copyMask.member(LA(1).getType()) ) {
+
+      if (copyMask.member(LA(1).getType())) {
+
+        // copy the token and link it backwards
+        if (hiddenCopy != null) {
+          linkAndCopyToken(hiddenCopy, monitored);
+        } else {
+          linkAndCopyToken(prev, monitored);
+        }
+
+        // we now need to parse it as a monitored token, so we return, which
+        // avoids the consume() call at the end of this loop.  the next call
+        // will parse it as a monitored token.
+        //System.err.println("returned: " + monitored.toString());
+        return monitored;
+
+      } else if (hideMask.member(LA(1).getType())) {
+
+        // attach the hidden token to the monitored in a chain
+        // link forwards
+        prev.setHiddenAfter(LA(1));
+
+        // link backwards
+        if (prev != monitored) { //hidden cannot point to monitored tokens
+          LA(1).setHiddenBefore(prev);
+        } else if (hiddenCopy != null) {
+          hiddenCopy.setHiddenAfter(LA(1));
+          LA(1).setHiddenBefore(hiddenCopy);
+          hiddenCopy = null;
+        }
+
+        //System.err.println("hidden: " + prev.getHiddenAfter().toString() + "\" after: " + prev.toString());
+        prev = lastHiddenToken = LA(1);
+      }
+
+      consume();
+    }
+
+    // remember the last hidden token for next time around
+    if (hiddenCopy != null) {
+      lastHiddenToken = hiddenCopy;
+      hiddenCopy = null;
+    }
+
+    //System.err.println("returned: " + monitored.toString());
+    return monitored;
+  }
+}