中文分词代码，此代码为作者多年经验总结，以前发表过VB，PB版本

/*

* created by yzh 2004.5.12

* 请大家引用时保留这段作者声明，此代码为开源代码；使用不受限制。

* 中文分词代码

*此代码为作者多年经验总结，以前发表过VB，PB版本

*/

import java.io.BufferedReader;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.util.Locale;

import java.util.TreeMap;

import java.util.TreeSet;

public class ChineseSegmenter {

private static ChineseSegmenter segmenter = null;

// private Hashtable zhwords;

private TreeMap zhwords;

private TreeSet cforeign, cnumbers;

// Char form

public final static int TRAD = 0;

public final static int SIMP = 1;

public final static int BOTH = 2;

// Charform is TRAD, SIMP or BOTH

private ChineseSegmenter(int charform, boolean loadwordfile) {

cforeign = new TreeSet();

cnumbers = new TreeSet();

if (charform == SIMP) {

loadset(cnumbers, "data/snumbers_u8.txt");

loadset(cforeign, "data/sforeign_u8.txt");

} else if (charform == TRAD) {

loadset(cnumbers, "data/tnumbers_u8.txt");

loadset(cforeign, "data/tforeign_u8.txt");

} else { // BOTH

loadset(cnumbers, "data/snumbers_u8.txt");

loadset(cforeign, "data/sforeign_u8.txt");

loadset(cnumbers, "data/tnumbers_u8.txt");

loadset(cforeign, "data/tforeign_u8.txt");

}

// zhwords = new Hashtable(120000);

zhwords = new TreeMap();

if (!loadwordfile) {

return;

}

String newword = null;

try {

InputStream worddata = null;

if (charform == SIMP) {

worddata = getClass().getResourceAsStream("simplexu8.txt");

} else if (charform == TRAD) {

worddata = getClass().getResourceAsStream("tradlexu8.txt");

} else if (charform == BOTH) {

worddata = getClass().getResourceAsStream("bothlexu8.txt");

}

BufferedReader in = new BufferedReader(new InputStreamReader(

worddata, "UTF8"));

while ((newword = in.readLine()) != null) {

if ((newword.indexOf("#") == -1) && (newword.length() < 5)) {

zhwords.put(newword.intern(), "1");

if (newword.length() == 3) {

if (zhwords.containsKey(newword.substring(0, 2)

.intern()) == false) {

zhwords.put(newword.substring(0, 2).intern(), "2");

}

}

if (newword.length() == 4) {

if (zhwords.containsKey(newword.substring(0, 2)

.intern()) == false) {

zhwords.put(newword.substring(0, 2).intern(), "2");

}

if (zhwords.containsKey(newword.substring(0, 3)

.intern()) == false) {

zhwords.put(newword.substring(0, 3).intern(), "2");

}

}

}

}

in.close();

} catch (IOException e) {

e.printStackTrace();

}

}

public synchronized static void reset() {

ChineseSegmenter.segmenter = null;

}

public synchronized static ChineseSegmenter getGBSegmenter() {

Locale.setDefault(Locale.SIMPLIFIED_CHINESE);

if (ChineseSegmenter.segmenter == null) {

ChineseSegmenter.segmenter = new ChineseSegmenter(ChineseSegmenter.SIMP, true);

}

return ChineseSegmenter.segmenter;

}

public synchronized static ChineseSegmenter getBig5Segmenter() {

Locale.setDefault(Locale.TRADITIONAL_CHINESE);

if (ChineseSegmenter.segmenter == null) {

ChineseSegmenter.segmenter = new ChineseSegmenter(ChineseSegmenter.TRAD, true);

}

return ChineseSegmenter.segmenter;

}

private void loadset(TreeSet targetset, String sourcefile) {

String dataline;

try {

InputStream setdata = getClass().getResourceAsStream(sourcefile);

BufferedReader in = new BufferedReader(new InputStreamReader(

setdata, "UTF-8"));

while ((dataline = in.readLine()) != null) {

if ((dataline.indexOf("#") > -1) || (dataline.length() == 0)) {

continue;

}

targetset.add(dataline.intern());

}

in.close();

} catch (Exception e) {

System.err.println("Exception loading data file" + sourcefile + " "

+ e);

e.printStackTrace();

}

}

public boolean isNumber(String testword) {

boolean result = true;

for (int i = 0; i < testword.length(); i++) {

if (cnumbers.contains(testword.substring(i, i + 1).intern()) == false) {

result = false;

break;

}

}

return result;

}

public boolean isAllForeign(String testword) {

boolean result = true;

for (int i = 0; i < testword.length(); i++) {

if (cforeign.contains(testword.substring(i, i + 1).intern()) == false) {

result = false;

break;

}

}

return result;

}

public boolean isNotCJK(String testword) {

boolean result = true;

for (int i = 0; i < testword.length(); i++) {

if (Character.UnicodeBlock.of(testword.charAt(i)) == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS) {

result = false;

break;

}

}

return result;

}

public String segmentLine(String cline, String separator) {

StringBuffer currentword = new StringBuffer();

StringBuffer outline = new StringBuffer();

int i, clength;

char currentchar;

// separator = " ";

clength = cline.length();

for (i = 0; i < clength; i++) {

currentchar = cline.charAt(i);

if (Character.UnicodeBlock.of(currentchar) == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS

|| isNumber(cline.substring(i, i + 1)) == true) {

// Character in CJK block

if (currentword.length() == 0) { // start looking for next

// word

if (i > 0

&& (Character.isWhitespace(cline.charAt(i - 1)) == false)) {

outline.append(separator);

}

currentword.append(currentchar);

} else {

if (zhwords.containsKey(new String(currentword.toString()

+ currentchar).intern()) == true

&& ((String) (zhwords.get(new String(currentword

.toString()

+ currentchar).intern()))).equals("1") == true) {

// word is in lexicon

currentword.append(currentchar);

} else if (isAllForeign(currentword.toString())

&& cforeign.contains(new String(

new char[] { currentchar }).intern())

&& i + 2 < clength

&& (zhwords.containsKey(cline.substring(i, i + 2)

.intern()) == false)) {

// Possible a transliteration of a foreign name

currentword.append(currentchar);

} else if (isNumber(currentword.toString())

&& cnumbers.contains(new String(

new char[] { currentchar }).intern())

/*

* && (i + 2 < clength) &&

* (zhwords.containsKey(cline.substring(i, i+2).intern()) ==

* false)

*/) {

// Put all consecutive number characters together

currentword.append(currentchar);

} else if ((zhwords.containsKey(new String(currentword

.toString()

+ currentchar).intern()))

&& (((String) (zhwords.get(new String(currentword

.toString()

+ currentchar).intern()))).equals("2") == true)

&& i + 1 < clength

&& (zhwords.containsKey(new String(currentword

.toString()

+ currentchar + cline.charAt(i + 1))

.intern()) == true)) {

// Starts a word in the lexicon

currentword.append(currentchar);

} else { // Start anew

outline.append(currentword.toString());

if (Character.isWhitespace(currentchar) == false) {

outline.append(separator);

}

currentword.setLength(0);

currentword.append(currentchar);

}

}

} else { // Not chinese character

// System.err.println("not cjk");

if (currentword.length() > 0) {

outline.append(currentword.toString());

if (Character.isWhitespace(currentchar) == false) {

outline.append(separator);

}

currentword.setLength(0);

}

outline.append(currentchar);

}

}

outline.append(currentword.toString());

return outline.toString();

// return offsets;

}

public static void main(String[] args) throws Exception {

ChineseSegmenter seg = ChineseSegmenter.getGBSegmenter();

System.out.println(seg.segmentLine("Some string in chinese.", " "));

}

}