StreamTokenizer.hpp

Go to the documentation of this file.

#pragma once
#ifndef BIL_STREAMTOKENIZER_HPP
#define BIL_STREAMTOKENIZER_HPP

#include <cstring>
#include <istream>


namespace bil {

    class StreamTokenizer {
    public:

        /**********************************************************************/
        /* CONSTRUCTION / DESTRUCTION                                         */
        /**********************************************************************/

        explicit StreamTokenizer(std::istream& inputStream);

        StreamTokenizer::~StreamTokenizer();


        /**********************************************************************/
        /* INPUT STREAM                                                       */
        /**********************************************************************/

        void inputStream(std::istream& inputStream);


        /**********************************************************************/
        /* TOKENIZER STATE                                                    */
        /**********************************************************************/

        void reset();


        /**********************************************************************/
        /* SYNTAX SETUP                                                       */
        /**********************************************************************/

        void separatorChar(char c);

        void separatorChars(char firstChar, char lastChar);

        void wordChar(char c);

        void wordChars(char firstChar, char lastChar);

        void whitespaceChar(char c);

        void commentChar(char c);

        void resetSyntax();


        /**********************************************************************/
        /* TOKEN RETRIEVAL                                                    */
        /**********************************************************************/

        typedef unsigned token_t;

        const static token_t TT_NONE = 0;

        const static token_t TT_EOF = 1;

        const static token_t TT_SEPARATOR = 2;

        const static token_t TT_WORD = 3;

        token_t nextToken();


        /**********************************************************************/
        /* CURRENT TOKEN                                                      */
        /**********************************************************************/

        token_t tokenType() const;


        char separatorToken() const;


        const char* wordToken() const;

        size_t wordTokenSize() const;


        bool uintToken(unsigned& val) const;


    private:

        StreamTokenizer(const StreamTokenizer&);
        StreamTokenizer& operator=(const StreamTokenizer&);

        size_t checkBuffer();

        std::istream* m_inputStream;
        char* m_buffer;
        size_t m_bufferSize;
        size_t m_readIndex;

        unsigned char m_syntaxTable[0xff];

        token_t m_tokenType;
        char m_separatorToken;
        char* m_wordBuffer;
        size_t m_wordBufferSize;

    };

}

#endif