CLucene - a full-featured, c++ search engine
API Documentation
00001 /*------------------------------------------------------------------------------ 00002 * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team 00003 * 00004 * Distributable under the terms of either the Apache License (Version 2.0) or 00005 * the GNU Lesser General Public License, as specified in the COPYING file. 00006 ------------------------------------------------------------------------------*/ 00007 #ifndef _lucene_analysis_standard_StandardTokenizer 00008 #define _lucene_analysis_standard_StandardTokenizer 00009 00010 #if defined(_LUCENE_PRAGMA_ONCE) 00011 # pragma once 00012 #endif 00013 00014 #include "../AnalysisHeader.h" 00015 #include "../Analyzers.h" 00016 #include "StandardTokenizerConstants.h" 00017 #include "CLucene/util/StringBuffer.h" 00018 #include "CLucene/util/FastCharStream.h" 00019 #include "CLucene/util/Reader.h" 00020 00021 00022 CL_NS_DEF2(analysis,standard) 00023 00024 00040 class StandardTokenizer: public Tokenizer { 00041 private: 00042 int32_t rdPos; 00043 int32_t tokenStart; 00044 00045 // Advance by one character, incrementing rdPos and returning the character. 00046 int readChar(); 00047 // Retreat by one character, decrementing rdPos. 00048 void unReadChar(); 00049 00050 // createToken centralizes token creation for auditing purposes. 00051 //Token* createToken(CL_NS(util)::StringBuffer* sb, TokenTypes tokenCode); 00052 inline bool setToken(Token* t, CL_NS(util)::StringBuffer* sb, TokenTypes tokenCode); 00053 00054 bool ReadDotted(CL_NS(util)::StringBuffer* str, TokenTypes forcedType,Token* t); 00055 00056 public: 00057 CL_NS(util)::FastCharStream* rd; 00058 00059 // Constructs a tokenizer for this Reader. 00060 StandardTokenizer(CL_NS(util)::Reader* reader); 00061 00062 ~StandardTokenizer(); 00063 00067 bool next(Token* token); 00068 00069 // Reads for number like "1"/"1234.567", or IP address like "192.168.1.2". 00070 bool ReadNumber(const TCHAR* previousNumber, const TCHAR prev, Token* t); 00071 00072 bool ReadAlphaNum(const TCHAR prev, Token* t); 00073 00074 // Reads for apostrophe-containing word. 00075 bool ReadApostrophe(CL_NS(util)::StringBuffer* str, Token* t); 00076 00077 // Reads for something@... it may be a COMPANY name or a EMAIL address 00078 bool ReadAt(CL_NS(util)::StringBuffer* str, Token* t); 00079 00080 // Reads for COMPANY name like AT&T. 00081 bool ReadCompany(CL_NS(util)::StringBuffer* str, Token* t); 00082 00083 // Reads CJK characters 00084 bool ReadCJK(const TCHAR prev, Token* t); 00085 }; 00086 00087 CL_NS_END2 00088 #endif