279,285c279
<       if (
< //          if there is text beyond the last token considered..
<           (lastEndOffset < text.length())
<           &&
< //          and that text is not too large...
<           (text.length()<= maxDocCharsToAnalyze)
<         )
---
>       if (lastEndOffset < text.length())
287a282,298
>         if(textFragmenter instanceof SimpleFragmenter)
>         {
>           SimpleFragmenter simpleFragmenter = (SimpleFragmenter) textFragmenter;
>           int remain =simpleFragmenter.getFragmentSize() -(newText.length() - currentFrag.textStartPos);
>           if(remain > 0 )
>           {
>             int endIndex = lastEndOffset + remain;
>             if (endIndex > text.length()) {
>               endIndex = text.length();
>             }
>             // newText.append(encoder.encodeText(text.substring(lastEndOffset)));
>             newText.append(encoder.encodeText(text.substring(lastEndOffset,
>                 endIndex)));
>           }
>         }
>         else
>         {
288a300
>         }