perl -正向最大匹配 转自Sighan 提供的FMM程序

#!/usr/bin/perl -w

# GBK编码,参数一 词典 参数二 待分文本

#转自Sighan 提供的FMM程序 ，不是原创

#所以把人家的声明都放在下面了

###########################################################################

# #

# SIGHAN #

# Copyright (c) 2003 #

# All Rights Reserved. #

# #

# Permission is hereby granted, free of charge, to use and distribute #

# this software and its documentation without restriction, including #

# without limitation the rights to use, copy, modify, merge, publish, #

# distribute, sublicense, and/or sell copies of this work, and to #

# permit persons to whom this work is furnished to do so, subject to #

# the following conditions: #

# 1. The code must retain the above copyright notice, this list of #

# conditions and the following disclaimer. #

# 2. Any modifications must be clearly marked as such. #

# 3. Original authors' names are not deleted. #

# 4. The authors' names are not used to endorse or promote products #

# derived from this software without specific prior written #

# permission. #

# #

# SIGHAN AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES #

# WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF #

# MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL SIGHAN NOR THE #

# CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL #

# DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA #

# OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER #

# TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR #

# PERFORMANCE OF THIS SOFTWARE. #

# #

###########################################################################

# #

# Author: Richard Sproat (rws@research.att.com) #

# #

###########################################################################

$USAGE = "Usage:\t$0 dictionary\n\t";

if (@ARGV < 1) {print "$USAGE\n"; exit;}

%dict = ();

$maxwlen = 0;

open (S, $ARGV[0]) or die "$ARGV[0]: $!\n";

while (<S>) {

chop;

$dict{$_} = 1;

my $l = length($_);

$maxwlen = $l if $l > $maxwlen;

}

close (S);

shift @ARGV;

$n = 0;

while (<>) {

chop;

s/\s*//g;

my $text = $_;

#print "$text\t";

while ($text ne "") {

$sub = substr($text, 0, $maxwlen);

while ($sub ne "") {

if ($dict{$sub}) {

print "$sub\/";

for (my $i = 0; $i < length($sub); ++$i) {

$text =~ s/^.//;

}

last;

}

$sub =~ s/.$//;

}

if ($sub eq "") {

if ($text =~ /^([\x21-\x7e])/) {

print "$1\/";

$text =~ s/^.//;

}

elsif ($text =~ /^([^\x21-\x7e].)/) {

print "$1\/";

$text =~ s/^..//;

}

else { ## shouldn't happen

print STDERR "Oops: shouldn't be here: $n\n";

print "$1\/";

$text =~ s/^.//;

}

}

}

print "\n";

++$n;

}

exit(0);