CLucene - a full-featured, c++ search engine
API Documentation

---

Analyzers.h

Go to the documentation of this file.

/*------------------------------------------------------------------------------
* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team
* 
* Distributable under the terms of either the Apache License (Version 2.0) or 
* the GNU Lesser General Public License, as specified in the COPYING file.
------------------------------------------------------------------------------*/
#ifndef _lucene_analysis_Analyzers_
#define _lucene_analysis_Analyzers_

#if defined(_LUCENE_PRAGMA_ONCE)
# pragma once
#endif

#include "CLucene/util/Reader.h"
#include "AnalysisHeader.h"
#include "CLucene/util/Misc.h"

CL_NS_DEF(analysis)


class CharTokenizer:public Tokenizer {
private:
        int32_t offset, bufferIndex, dataLen;
        TCHAR buffer[LUCENE_MAX_WORD_LEN+1];
        const TCHAR* ioBuffer;
protected:
    
        virtual bool isTokenChar(const TCHAR c) const = 0;

        virtual TCHAR normalize(const TCHAR c) const;

public:
        CharTokenizer(CL_NS(util)::Reader* in);
        virtual ~CharTokenizer(){
        }
        bool next(Token* token);
};


class LetterTokenizer:public CharTokenizer {
public:
        // Construct a new LetterTokenizer. 
        LetterTokenizer(CL_NS(util)::Reader* in):
        CharTokenizer(in) {}

    ~LetterTokenizer(){}
protected:
        bool isTokenChar(const TCHAR c) const;
};



class LowerCaseTokenizer:public LetterTokenizer {
public:
        LowerCaseTokenizer(CL_NS(util)::Reader* in):
        LetterTokenizer(in) {}

    ~LowerCaseTokenizer(){}
protected:
        TCHAR normalize(const TCHAR chr) const;
};


class WhitespaceTokenizer: public CharTokenizer {
public:
        WhitespaceTokenizer(CL_NS(util)::Reader* in):CharTokenizer(in) {}
        ~WhitespaceTokenizer(){}
protected:
        bool isTokenChar(const TCHAR c) const;
};


class WhitespaceAnalyzer: public Analyzer {
 public:
  TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader);
  ~WhitespaceAnalyzer(){}
};

class SimpleAnalyzer: public Analyzer {
public:
        TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader);
        ~SimpleAnalyzer(){}
};



class LowerCaseFilter: public TokenFilter {
public:
        LowerCaseFilter(TokenStream* in, bool deleteTokenStream):TokenFilter(in,deleteTokenStream) {}
        ~LowerCaseFilter(){}
        bool next(Token* token);
};


class StopFilter: public TokenFilter {
private:
        //bvk: i found this to work faster with a non-hash table. the number of items
        //in the stop table is not like to make it worth having hashing.
        CL_NS(util)::CLSetList<const TCHAR*>* table;
public:
        // Constructs a filter which removes words from the input
        //      TokenStream that are named in the array of words. 
        StopFilter(TokenStream* in, bool deleteTokenStream, const TCHAR** stopWords);

        ~StopFilter(){}

        StopFilter(TokenStream* in, bool deleteTokenStream, CL_NS(util)::CLSetList<const TCHAR*>* stopTable):
                TokenFilter(in, deleteTokenStream),
                table(stopTable)
        {} 
          
        
        static void fillStopTable(CL_NS(util)::CLSetList<const TCHAR*>* stopTable,
                                      const TCHAR** stopWords);

        bool next(Token* token);
};




class StopAnalyzer: public Analyzer {
    CL_NS(util)::CLSetList<const TCHAR*> stopTable;

public:
    StopAnalyzer();
    ~StopAnalyzer();
    
    StopAnalyzer( const TCHAR** stopWords );
    TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader);
        
    static const TCHAR* ENGLISH_STOP_WORDS[];
};



class PerFieldAnalyzerWrapper : public Analyzer {
private:
    Analyzer* defaultAnalyzer;
    CL_NS(util)::CLHashMap<const TCHAR*, Analyzer*, CL_NS(util)::Compare::TChar,
    CL_NS(util)::Equals::TChar, CL_NS(util)::Deletor::tcArray,CL_NS(util)::Deletor::Void<Analyzer> > analyzerMap;
public:
    PerFieldAnalyzerWrapper(Analyzer* defaultAnalyzer);
    ~PerFieldAnalyzerWrapper();
    
    void addAnalyzer(const TCHAR* fieldName, Analyzer* analyzer);
    TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader);
};


class ISOLatin1AccentFilter: public TokenFilter {
public:
        ISOLatin1AccentFilter(TokenStream* input, bool deleteTs):
                TokenFilter(input,deleteTs)
        {
        }
        
        bool next(Token* token);
};


class KeywordTokenizer: public Tokenizer {
private:
    LUCENE_STATIC_CONSTANT(int, DEFAULT_BUFFER_SIZE = 256);
    bool done;
    int bufferSize;
public:
    KeywordTokenizer(CL_NS(util)::Reader* input, int bufferSize=-1);
    virtual ~KeywordTokenizer();
    bool next(Token* token);
};

class KeywordAnalyzer: public Analyzer {
public:
    TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader);
    virtual ~KeywordAnalyzer(){}
};

    
class LengthFilter: public TokenFilter {
private:
    int _min;
    int _max;
public:
    LengthFilter(TokenStream* in, int _min, int _max);
    
    bool next(Token* token);
};


CL_NS_END
#endif

---

clucene.sourceforge.net