|

Utf8Tokenizer.hpp

Go to the documentation of this file.

#ifndef IONFLUX_TOOLS_UTF8TOKENIZER
#define IONFLUX_TOOLS_UTF8TOKENIZER
/* ==========================================================================
 * Ionflux Tools
 * Copyright (c) 2005 Joern P. Meier
 * mail@ionflux.org
 * --------------------------------------------------------------------------
 * Utf8Tokenizer.hpp                 Tokenizer with UTF-8 support.
 * ==========================================================================
 * 
 * This file is part of Ionflux Tools.
 * 
 * Ionflux Tools is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the 
 * Free Software Foundation; either version 2 of the License, or (at your 
 * option) any later version.
 * 
 * Ionflux Tools is distributed in the hope that it will be useful, but 
 * WITHOUT ANY WARRANTY; without even the implied warranty of 
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU 
 * General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License along 
 * with Ionflux Tools; if not, write to the Free Software Foundation, Inc.,
 * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 * 
 * ========================================================================== */

#include "ionflux/Utf8TokenTypeMap.hpp"
#include "ionflux/ManagedObject.hpp"

namespace Ionflux
{

namespace Tools
{

struct Utf8Token
{
    int typeID;
    std::string value;
};

class Utf8TokenizerClassInfo
: public Ionflux::Tools::ClassInfo
{
    public:
        Utf8TokenizerClassInfo();
        virtual ~Utf8TokenizerClassInfo() { };
};

class Utf8Tokenizer
: public Ionflux::Tools::ManagedObject
{
    private:
        
    protected:
        std::vector<unsigned int> theInput;
        std::vector<unsigned int> quoteChars;
        unsigned int currentPos;
        unsigned int currentTokenPos;
        unsigned int currentQuoteChar;
        Utf8TokenTypeMap* typeMap;
        Utf8Token currentToken;
        bool extractQuoted;
        bool extractEscaped;
        
    public:
        static const Utf8TokenType TT_INVALID;
        static const Utf8TokenType TT_NONE;
        static const Utf8TokenType TT_DEFAULT;
        static const Utf8TokenType TT_QUOTED;
        static const Utf8TokenType TT_ESCAPED;
        static const Utf8TokenType TT_LINEAR_WHITESPACE;
        static const Utf8TokenType TT_LINETERM;
        static const Utf8TokenType TT_IDENTIFIER;
        static const Utf8TokenType TT_NUMBER;
        static const Utf8TokenType TT_ALPHA;
        static const Utf8TokenType TT_DEFAULT_SEP;
        static const Utf8TokenType TT_LATIN;
        static const Utf8Token TOK_INVALID;
        static const Utf8Token TOK_NONE;
        static const std::string QUOTE_CHARS;
        static const unsigned int ESCAPE_CHAR;
        static const Utf8TokenizerClassInfo utf8TokenizerClassInfo;
        static const Ionflux::Tools::ClassInfo* CLASS_INFO;
        
        Utf8Tokenizer();
        
        Utf8Tokenizer(const std::string& initInput);
        
        Utf8Tokenizer(const std::vector<Utf8TokenType>& initTokenTypes, const 
        std::string& initInput = "");
        
        virtual ~Utf8Tokenizer();
        
        virtual void reset();
        
        virtual void clearTokenTypes();
        
        virtual void useDefaultTokenTypes();
        
        virtual void addDefaultTokenType();
        
        virtual void setTokenTypes(const std::vector<Utf8TokenType>& 
        newTokenTypes);
        
        virtual void addTokenTypes(const std::vector<Utf8TokenType>& 
        newTokenTypes);
        
        virtual void addTokenType(const Utf8TokenType& newTokenType);
        
        virtual void setInput(const std::string& newInput);
        
        virtual void setInput(const std::vector<unsigned int>& newInput);
        
        virtual Utf8Token getNextToken(Utf8TokenTypeMap* otherTypeMap = 0);
        
        virtual Utf8Token getCurrentToken();
        
        virtual int getCurrentTokenType();
        
        virtual unsigned int getCurrentPos();
        
        virtual unsigned int getCurrentTokenPos();
        
        virtual unsigned int getQuoteChar();
        
        static bool isValid(const Utf8Token& checkToken);
        
        virtual void setExtractQuoted(bool newExtractQuoted);
        
        virtual bool getExtractQuoted() const;
        
        virtual void setExtractEscaped(bool newExtractEscaped);
        
        virtual bool getExtractEscaped() const;
};

}

}

#endif

---