diff options
Diffstat (limited to 'corpus/support/utf8-normalize.sh')
| -rwxr-xr-x | corpus/support/utf8-normalize.sh | 3 |
1 files changed, 2 insertions, 1 deletions
diff --git a/corpus/support/utf8-normalize.sh b/corpus/support/utf8-normalize.sh index 2f347854..c85ae9f7 100755 --- a/corpus/support/utf8-normalize.sh +++ b/corpus/support/utf8-normalize.sh @@ -25,7 +25,8 @@ else fi fi -perl -e 'while(<>){s/\r\n*/\n/g; print;}' | $CMD | /usr/bin/perl -w -e ' +perl -e '$|++; while(<>){s/\r\n*/\n/g; print;}' | $CMD | /usr/bin/perl -w -e ' + $|++; while (<>) { chomp; s/[\x00-\x1F]+/ /g; |