diff options
author | Chris Dyer <cdyer@cs.cmu.edu> | 2012-07-28 12:11:30 -0400 |
---|---|---|
committer | Chris Dyer <cdyer@cs.cmu.edu> | 2012-07-28 12:11:30 -0400 |
commit | 934e55dc12c3f374684bc6a0797e6f85c7abb85a (patch) | |
tree | ab9f49ddbaf4518187abeea52eb3fccdcae3f2ed /corpus/utf8-normalize.sh | |
parent | b317e0efd2398d75d70e027bb1e2cf442e683981 (diff) |
a couple of tools for cleaning corpora
Diffstat (limited to 'corpus/utf8-normalize.sh')
-rwxr-xr-x | corpus/utf8-normalize.sh | 25 |
1 files changed, 25 insertions, 0 deletions
diff --git a/corpus/utf8-normalize.sh b/corpus/utf8-normalize.sh new file mode 100755 index 00000000..dcf8bc59 --- /dev/null +++ b/corpus/utf8-normalize.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# This script uses ICU uconv (http://site.icu-project.org/), if it's available +# to normalize UTF8 text into a standard form. For information about this +# process, refer to http://en.wikipedia.org/wiki/Unicode_equivalence#Normalization +# Escape characters between 0x00-0x1F are removed + +if which uconv > /dev/null +then + CMD="uconv -f utf8 -t utf8 -x Any-NFKC --callback skip" +else + echo "Cannot find ICU uconv (http://site.icu-project.org/) ... falling back to iconv. Normalization NOT taking place." 1>&2 + CMD="iconv -f utf8 -t utf8 -c" +fi + +$CMD | /usr/bin/perl -w -e ' + while (<>) { + chomp; + s/[\x00-\x1F]+/ /g; + s/ +/ /g; + s/^ //; + s/ $//; + print "$_\n"; + }' + |