|

Tokenizer.hpp

Go to the documentation of this file.

#ifndef IONFLUX_TOOLS_TOKENIZER
#define IONFLUX_TOOLS_TOKENIZER
/* ==========================================================================
 * Ionflux Tools
 * Copyright (c) 2004 Joern P. Meier
 * mail@ionflux.org
 * --------------------------------------------------------------------------
 * Tokenizer.hpp                    Generic byte string tokenizer.
 * ==========================================================================
 * 
 * This file is part of Ionflux Tools.
 * 
 * Ionflux Tools is free software; you can redistribute it and/or modify it 
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at  your option)
 * any later version.
 * 
 * Ionflux Tools is distributed in the hope that it will be useful, but 
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with Ionflux Tools; if not, write to the Free Software Foundation, 
 * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 * 
 * ========================================================================== */

#include <string>
#include <iostream>
#include <vector>
#include "ionflux/tools.hpp"
#include "ionflux/TokenTypeMap.hpp"

namespace Ionflux
{

namespace Tools
{


struct TokenType
{
    int typeID;
    std::string validChars;
    bool invert;
    int maxChars;
    
    static const int USERTYPE_ID;
};

struct Token
{
    int typeID;
    std::string value;
};

class TokenTypeMap;

class Tokenizer
{
    protected:
        std::string theInput;
        unsigned int currentPos;
        unsigned int currentTokenPos;
        Token currentToken;
        bool extractQuoted;
        char currentQuoteChar;
        bool extractEscaped;
        TokenTypeMap *typeMap;
        
    public:
        TokenType TT_ANYTHING;
        static const TokenType TT_INVALID;
        static const TokenType TT_NONE;
        static const TokenType TT_QUOTED;
        static const TokenType TT_ESCAPED;
        static const TokenType TT_WHITESPACE;
        static const TokenType TT_LINETERM;
        static const TokenType TT_NUMBER;
        static const TokenType TT_ALPHA;
        static const TokenType TT_DEFAULT_SEP;
        static const TokenType TT_IDENTIFIER;
        static const Token TOK_INVALID;
        static const Token TOK_NONE;
        static const int TT_ANYTHING_TYPE_ID;
        static const std::string QUOTE_CHARS;
        static const char ESCAPE_CHAR;
        
        Tokenizer();
        
        Tokenizer(const std::string& initInput);
        
        Tokenizer(const std::vector<TokenType>& initTokenTypes);
        
        Tokenizer(const std::vector<TokenType>& initTokenTypes, 
            const std::string& initInput);
        
        virtual ~Tokenizer();
        
        virtual void clearTokenTypes();
        
        virtual void useDefaultTokenTypes();
        
        virtual void setTokenTypes(const std::vector<TokenType>& newTokenTypes);
        
        virtual void addTokenType(const TokenType& newTokenType);
        
        virtual void addTokenTypes(const std::vector<TokenType>& newTokenTypes);
        
        virtual void setInput(const std::string& newInput);
        
        virtual Token nextToken();
        
        virtual Token getNextToken(const TokenTypeMap &otherMap);
        
        virtual Token getNextToken();
        
        virtual Token getCurrentToken();
        
        virtual int getCurrentTokenType();
        
        virtual void reset();
        
        virtual void setTokenTypeAnything();
        
        virtual void setExtractQuoted(bool newExtractQuoted);
        
        virtual void setExtractEscaped(bool newExtractEscaped);
        
        virtual unsigned int getCurrentPos();
        
        virtual unsigned int getCurrentTokenPos();
        
        virtual char getQuoteChar();
        
        static bool isOneOf(char c, const std::string& testChars, bool invert);
        
        static bool isValid(Token& token);
};


}

}

#endif

---