[Parser] tokenContext auto-removes certain tokens

include/occa/c/types.h

@@ -100,7 +100,7 @@ extern const occaUDim_t occaAllBytes;
 OCCA_LFUNC bool OCCA_RFUNC occaIsUndefined(occaType value);
 OCCA_LFUNC bool OCCA_RFUNC occaIsDefault(occaType value);
 
-OCCA_LFUNC occaType OCCA_RFUNC occaPtr(void *value);
+OCCA_LFUNC occaType OCCA_RFUNC occaPtr(const void *value);
 
 OCCA_LFUNC occaType OCCA_RFUNC occaBool(bool value);
 

include/occa/c/types.hpp

@@ -49,6 +49,7 @@ namespace occa {
     occaType newOccaType(const TM &value);
 
     occaType newOccaType(void *value);
+    occaType newOccaType(const void *value);
 
     template <>
     occaType newOccaType(const occa::primitive &value);

include/occa/lang/parser.hpp

@@ -51,10 +51,10 @@ namespace occa {
       statementContext_t smntContext;
       statementPeeker_t smntPeeker;
 
-      bool ignoringComments;
       bool checkSemicolon;
 
       unknownToken defaultRootToken;
+      statementPtrVector comments;
       attributeTokenMap attributes;
 
       bool success;
@@ -114,6 +114,11 @@ namespace occa {
       exprNode* getExpression(const int start,
                               const int end);
 
+      void loadComments();
+      void loadComments(const int start,
+                        const int end);
+      void pushComments();
+
       void loadAttributes(attributeTokenMap &attrs);
 
       attribute_t* getAttribute(const std::string &name);
@@ -165,8 +170,8 @@ namespace occa {
 
       void loadAllStatements();
 
+      statement_t* loadNextStatement();
       statement_t* getNextStatement();
-      statement_t* getNextNonCommentStatement();
 
       statement_t* loadBlockStatement(attributeTokenMap &smntAttributes);
 
@@ -205,8 +210,6 @@ namespace occa {
 
       statement_t* loadClassAccessStatement(attributeTokenMap &smntAttributes);
 
-      statement_t* loadCommentStatement(attributeTokenMap &smntAttributes);
-
       statement_t* loadDirectiveStatement(attributeTokenMap &smntAttributes);
       statement_t* loadPragmaStatement(attributeTokenMap &smntAttributes);
 

include/occa/lang/tokenContext.hpp

@@ -36,6 +36,9 @@ namespace occa {
     class tokenContext_t {
     public:
       tokenVector tokens;
+      // Keep track of used tokens (e.g. not commnents)
+      intVector tokenIndices;
+
       intIntMap pairs;
       intVector semicolons;
       bool hasError;
@@ -48,7 +51,8 @@ namespace occa {
       ~tokenContext_t();
 
       void clear();
-      void setup();
+      void setup(const tokenVector &tokens_);
+      void setupTokenIndices();
 
       void findPairs();
       void findSemicolons();
@@ -72,16 +76,24 @@ namespace occa {
       void popAndSkip();
 
       int position() const;
+
       int size() const;
 
+      void getSkippedTokens(tokenVector &skippedTokens,
+                            const int start,
+                            const int end);
+
+      token_t* getToken(const int index);
+      token_t* getNativeToken(const int index);
+
+      void setToken(const int index,
+                    token_t *value);
+
       token_t* operator [] (const int index);
       tokenContext_t& operator ++ ();
       tokenContext_t& operator ++ (int);
       tokenContext_t& operator += (const int offset);
 
-      void setToken(const int index,
-                    token_t *value);
-
       token_t* end();
 
       token_t* getPrintToken(const bool atEnd);

src/c/types.cpp

@@ -31,6 +31,10 @@ namespace occa {
     }
 
     occaType newOccaType(void *value) {
+      return newOccaType((const void*) value);
+    }
+
+    occaType newOccaType(const void *value) {
       occaType oType;
       oType.magicHeader = OCCA_C_TYPE_MAGIC_HEADER;
       oType.type  = typeType::ptr;
@@ -670,7 +674,7 @@ OCCA_LFUNC bool OCCA_RFUNC occaIsDefault(occaType value) {
   return (value.type == occa::c::typeType::default_);
 }
 
-OCCA_LFUNC occaType OCCA_RFUNC occaPtr(void *value) {
+OCCA_LFUNC occaType OCCA_RFUNC occaPtr(const void *value) {
   return occa::c::newOccaType(value);
 }