前向最大中文分词代码java.docVIP

前向最大中文分词java源码 一．中文分词目前主要方法 1）基于词典的匹配： 前向最大匹配，后向最大匹配 2）基于字的标注： ??最大熵模型，条件随机场模型，感知器模型 3）其它方法 ??与词性标注结合，与句法分析结合 二．中文分词前向最大匹配java代码 package iim.segCn; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.TreeMap; import java.util.TreeSet; /** * ***************************************************** * 分词 * ***************************************************** * * @author * @version 1.100, 07/07/06 * @since JDK1.6 */ public class SegCn { private String separator= ; private static SegCn segmenter = null; private TreeMapString, Boolean cnWords; private TreeSetString cForeign, cNumbers; private SegCn() { cForeign = new TreeSetString(); cNumbers = new TreeSetString(); loadset(cNumbers, data/snumbers_u8.txt); loadset(cForeign, data/sforeign_u8.txt); System.out.print(Loading Lexicon); cnWords = new TreeMapString, Boolean(); String newword = null; try { InputStream worddata = null; worddata =new FileInputStream(new File(data/lexicon)); BufferedReader in = new BufferedReader(new InputStreamReader( worddata)); int i=0; while ((newword = in.readLine()) != null) { if((++i)%10000==0){ System.out.print(.); } if ((newword.indexOf(#) == -1) (newword.length() 5)) { cnWords.put(ern(), true); if (newword.length() == 3) { if (cnWords.containsKey(newword.substring(0, 2) .intern()) == false) { cnWords.put(newword.substring(0, 2).intern(), false); } } if (newword.length() == 4) { if (cnWords.containsKey(newword.substring(0, 2) .intern()) == false) { cnWords.put(newword.substring(0, 2).intern(), false); } if (cnWords.containsKey(newword.substring(0, 3) .intern()) == false) { cnWords.put(newword.substring(0, 3).intern(), false); } } } } in.close(); System.out.println();