CLucene - a full-featured, c++ search engine
API Documentation

---

TermVector.h

Go to the documentation of this file.

/*------------------------------------------------------------------------------
* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team
* 
* Distributable under the terms of either the Apache License (Version 2.0) or 
* the GNU Lesser General Public License, as specified in the COPYING file.
------------------------------------------------------------------------------*/
#ifndef _lucene_index_termvector_h
#define _lucene_index_termvector_h

#if defined(_LUCENE_PRAGMA_ONCE)
# pragma once
#endif

#include "FieldInfos.h"

CL_NS_DEF(index)

//predefine classes
struct TermVectorOffsetInfo;
class TermPositionVector; 

class TermFreqVector:LUCENE_BASE {
public:
        virtual ~TermFreqVector(){
        }

        virtual const TCHAR* getField() = 0;

        virtual int32_t size() = 0;

        virtual const TCHAR** getTerms() = 0;


        virtual const Array<int32_t>* getTermFrequencies() = 0;


        virtual int32_t indexOf(const TCHAR* term) = 0;


        virtual void indexesOf(const TCHAR** terms, const int32_t start, const int32_t len, Array<int32_t>& ret) = 0;

        virtual TermPositionVector* __asTermPositionVector()=0;
};


class TermVectorsWriter:LUCENE_BASE {
private:
        class TVField:LUCENE_BASE {
        public:
                int32_t number;
                int64_t tvfPointer;
                int32_t length;   // number of distinct term positions
                bool storePositions;
                bool storeOffsets;
         
                TVField(int32_t number, bool storePos, bool storeOff): 
                                tvfPointer(0),length(0){
                        this->number = number;
                        this->storePositions = storePos;
                        this->storeOffsets = storeOff;
                }
        ~TVField(){}
        };

        class TVTerm:LUCENE_BASE {
                const TCHAR* termText;
                int32_t termTextLen; //textlen cache
                
        public:
                TVTerm();
        ~TVTerm();
                
                int32_t freq;
                Array<int32_t>* positions;
                Array<TermVectorOffsetInfo>* offsets;

                const TCHAR* getTermText() const;
                size_t getTermTextLen();
                void setTermText(const TCHAR* val);
        };


        CL_NS(store)::IndexOutput* tvx, *tvd, *tvf;
        CL_NS(util)::CLVector<TVField*,CL_NS(util)::Deletor::Object<TVField> > fields;
        CL_NS(util)::CLVector<TVTerm*,CL_NS(util)::Deletor::Object<TVTerm> > terms;
        FieldInfos* fieldInfos;

        TVField* currentField;
        int64_t currentDocPointer;

        void addTermInternal(const TCHAR* termText, const int32_t freq, 
                Array<int32_t>* positions, Array<TermVectorOffsetInfo>* offsets);

        void writeField();
        void writeDoc();
  
        void openField(int32_t fieldNumber, bool storePositionWithTermVector, 
      bool storeOffsetWithTermVector);
public:
        LUCENE_STATIC_CONSTANT(int32_t, FORMAT_VERSION = 2);

        //The size in bytes that the FORMAT_VERSION will take up at the beginning of each file 
        LUCENE_STATIC_CONSTANT(int32_t, FORMAT_SIZE = 4);

        LUCENE_STATIC_CONSTANT(uint8_t, STORE_POSITIONS_WITH_TERMVECTOR = 0x1);
        LUCENE_STATIC_CONSTANT(uint8_t, STORE_OFFSET_WITH_TERMVECTOR = 0x2);
        
        static const char* LUCENE_TVX_EXTENSION;
        static const char* LUCENE_TVD_EXTENSION;
        static const char* LUCENE_TVF_EXTENSION;

        TermVectorsWriter(CL_NS(store)::Directory* directory, const char* segment,
                                                   FieldInfos* fieldInfos);

        ~TermVectorsWriter();
        void openDocument();
        void closeDocument();

        void close();
        bool isDocumentOpen() const;

        void openField(const TCHAR* field);

        void closeField();

        bool isFieldOpen() const;

        void addAllDocVectors(Array<TermFreqVector*>& vectors);

        void addTerm(const TCHAR* termText, int32_t freq, 
                Array<int32_t>* positions = NULL, Array<TermVectorOffsetInfo>* offsets = NULL);
};

class SegmentTermVector: public virtual TermFreqVector {
private:
        const TCHAR* field;
        TCHAR** terms;
        int32_t termsLen; //cache
        Array<int32_t>* termFreqs;

        int32_t binarySearch(TCHAR** a, const int32_t arraylen, const TCHAR* key) const;
public:
        //note: termFreqs must be the same length as terms
        SegmentTermVector(const TCHAR* field, TCHAR** terms, Array<int32_t>* termFreqs);
        virtual ~SegmentTermVector();

        const TCHAR* getField();
        TCHAR* toString() const;
        int32_t size();
        const TCHAR** getTerms();
        const Array<int32_t>* getTermFrequencies();
        int32_t indexOf(const TCHAR* termText);
        void indexesOf(const TCHAR** termNumbers, const int32_t start, const int32_t len, Array<int32_t>& ret);

        virtual TermPositionVector* __asTermPositionVector();
};



class TermVectorsReader:LUCENE_BASE {
private:
    FieldInfos* fieldInfos;
    
    CL_NS(store)::IndexInput* tvx;
    CL_NS(store)::IndexInput* tvd;
    CL_NS(store)::IndexInput* tvf;
    int64_t _size;
    
    int32_t tvdFormat;
    int32_t tvfFormat;
    
    
    int32_t checkValidFormat(CL_NS(store)::IndexInput* in);
    
        void readTermVectors(const TCHAR** fields, const int64_t* tvfPointers, const int32_t len, Array<TermFreqVector*>& _return);

    SegmentTermVector* readTermVector(const TCHAR* field, const int64_t tvfPointer);

    int64_t size();
  
  
        DEFINE_MUTEX(THIS_LOCK)
        TermVectorsReader(const TermVectorsReader& copy);
public:
        TermVectorsReader(CL_NS(store)::Directory* d, const char* segment, FieldInfos* fieldInfos);
        ~TermVectorsReader();

        void close();
        TermVectorsReader* clone() const;

        TermFreqVector* get(const int32_t docNum, const TCHAR* field);


        bool get(int32_t docNum, Array<TermFreqVector*>& result);
};


struct TermVectorOffsetInfo {
    int startOffset;
    int endOffset;
public:
        static Array<TermVectorOffsetInfo> EMPTY_OFFSET_INFO;
    TermVectorOffsetInfo();
    ~TermVectorOffsetInfo();
    TermVectorOffsetInfo(int32_t startOffset, int32_t endOffset);
    int32_t getEndOffset() const;
    void setEndOffset(int32_t endOffset);
    int32_t getStartOffset() const;
    void setStartOffset(int32_t startOffset);
    bool equals(TermVectorOffsetInfo* o);
    size_t hashCode() const;
};


class TermPositionVector: public virtual TermFreqVector {
public:

    virtual Array<int32_t>* getTermPositions(int32_t index) = 0;
  
     virtual Array<TermVectorOffsetInfo>* getOffsets(int32_t index) = 0;
     
     virtual ~TermPositionVector(){
         }
};


class SegmentTermPositionVector: public SegmentTermVector, public TermPositionVector {
protected:
        Array< Array<int32_t> >* positions;
        Array< Array<TermVectorOffsetInfo> >* offsets;
        static Array<int32_t> EMPTY_TERM_POS;
public:
        SegmentTermPositionVector(const TCHAR* field, TCHAR** terms, Array<int32_t>* termFreqs, Array< Array<int32_t> >* positions, Array< Array<TermVectorOffsetInfo> >* offsets);
        ~SegmentTermPositionVector();

        Array<TermVectorOffsetInfo>* getOffsets(int32_t index);

        Array<int32_t>* getTermPositions(int32_t index);

        const TCHAR* getField(){ return SegmentTermVector::getField(); }
        TCHAR* toString() const{ return SegmentTermVector::toString(); }
        int32_t size(){ return SegmentTermVector::size(); }
        const TCHAR** getTerms(){ return SegmentTermVector::getTerms(); }
        const Array<int32_t>* getTermFrequencies(){ return SegmentTermVector::getTermFrequencies(); }
        int32_t indexOf(const TCHAR* termText){ return SegmentTermVector::indexOf(termText); }
        void indexesOf(const TCHAR** termNumbers, const int32_t start, const int32_t len, Array<int32_t>& ret)
                { SegmentTermVector::indexesOf(termNumbers, start, len, ret); }

        virtual TermPositionVector* __asTermPositionVector();
};

CL_NS_END
#endif

---

clucene.sourceforge.net