CLucene - a full-featured, c++ search engine
API Documentation

---

AnalysisHeader.h

Go to the documentation of this file.

/*------------------------------------------------------------------------------
* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team
* 
* Distributable under the terms of either the Apache License (Version 2.0) or 
* the GNU Lesser General Public License, as specified in the COPYING file.
------------------------------------------------------------------------------*/
#ifndef _lucene_analysis_AnalysisHeader_
#define _lucene_analysis_AnalysisHeader_

#if defined(_LUCENE_PRAGMA_ONCE)
# pragma once
#endif

#include "CLucene/util/Reader.h"

CL_NS_DEF(analysis)
  


class Token:LUCENE_BASE{
private:
        int32_t _startOffset;                           // start in source text
        int32_t _endOffset;                               // end in source text
        const TCHAR* _type;                               // lexical type
        int32_t positionIncrement;
        size_t bufferTextLen;

public:
        #ifndef LUCENE_TOKEN_WORD_LENGTH
        TCHAR* _termText;                                 // the text of the term
        #else
        TCHAR _termText[LUCENE_TOKEN_WORD_LENGTH+1];                              // the text of the term
        #endif
        int32_t _termTextLen;
        static const TCHAR* defaultType;

        Token();
        ~Token();
        // Constructs a Token with the given text, start and end offsets, & type. 
        Token(const TCHAR* text, const int32_t start, const int32_t end, const TCHAR* typ=defaultType);
        void set(const TCHAR* text, const int32_t start, const int32_t end, const TCHAR* typ=defaultType);
        
        size_t bufferLength(){ return bufferTextLen; }
        void growBuffer(size_t size);
        
        /* Set the position increment.  This determines the position of this
        * token relative to the previous Token in a TokenStream, used in
        * phrase searching.
        *
        * The default value is 1.
        *
        * Some common uses for this are:
        *
        * - Set it to zero to put multiple terms in the same position.  This is
        * useful if, e.g., a word has multiple stems.  Searches for phrases
        * including either stem will match.  In this case, all but the first stem's
        * increment should be set to zero: the increment of the first instance
        * should be one.  Repeating a token with an increment of zero can also be
        * used to boost the scores of matches on that token.
        *
        * - Set it to values greater than one to inhibit exact phrase matches.
        * If, for example, one does not want phrases to match across removed stop
        * words, then one could build a stop word filter that removes stop words and
        * also sets the increment to the number of stop words removed before each
        * non-stop word.  Then exact phrase queries will only match when the terms
        * occur with no intervening stop words.
        */
        void setPositionIncrement(int32_t posIncr);
        int32_t getPositionIncrement() const;
        const TCHAR* termText() const;
        size_t termTextLength();
        void resetTermTextLen();
        void setText(const TCHAR* txt);

        int32_t startOffset() const { return _startOffset; }
        void setStartOffset(int32_t val){ _startOffset =val; }

        int32_t endOffset() const { return _endOffset; }
        void setEndOffset(int32_t val){ _endOffset =val; }

        // Returns this Token's lexical type.  Defaults to "word". 
        const TCHAR* type() const { return _type; } 
        void setType(const TCHAR* val) { _type = val; } 

        TCHAR* toString() const;

        class OrderCompare:LUCENE_BASE, public CL_NS(util)::Compare::_base //<Token*>
        {
        public:
                bool operator()( Token* t1, Token* t2 ) const;
        };
};

class TokenStream:LUCENE_BASE {
public:
        virtual bool next(Token* token) = 0;

        virtual void close() = 0;

        virtual ~TokenStream(){
        }

        /* This is for backwards compatibility only. You should pass the token you want to fill
         * to next(), this will save a lot of object construction and destructions.
         *  @deprecated. use next(token). Kept only to avoid breaking existing code.
         */
        _CL_DEPRECATED(next(Token)) Token* next();
};


class Analyzer:LUCENE_BASE{
public:
        virtual TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader)=0;
          
        virtual ~Analyzer(){
    }

        virtual int32_t getPositionIncrementGap(const TCHAR* fieldName);
};


class Tokenizer:public TokenStream {
protected:
        CL_NS(util)::Reader* input;

public:
        Tokenizer();
    Tokenizer(CL_NS(util)::Reader* _input);

    // ** By default, closes the input Reader. */
        virtual void close();
        virtual ~Tokenizer();
};

class TokenFilter:public TokenStream {
protected:
        TokenStream* input;
        bool deleteTokenStream;

        TokenFilter(TokenStream* in, bool deleteTS=false);
        virtual ~TokenFilter();
public:
        void close();
};

CL_NS_END
#endif

---

clucene.sourceforge.net