diff options
author | Michael Denkowski <mdenkows@cs.cmu.edu> | 2013-09-05 14:15:43 -0700 |
---|---|---|
committer | Michael Denkowski <mdenkows@cs.cmu.edu> | 2013-09-05 14:15:43 -0700 |
commit | c55f37fee9f43c0a13b47aac512804ecf9f5bd48 (patch) | |
tree | 5b29abbc119dab32d1e991fc4fd55594803b6850 /corpus/support/utf8-normalize-batch.pl | |
parent | 208fabfbbe19c1ba2ee744e9d16b54805ec8b141 (diff) |
Slower but correct (wrt buffered) unbuffered version.
Diffstat (limited to 'corpus/support/utf8-normalize-batch.pl')
-rwxr-xr-x | corpus/support/utf8-normalize-batch.pl | 28 |
1 files changed, 28 insertions, 0 deletions
diff --git a/corpus/support/utf8-normalize-batch.pl b/corpus/support/utf8-normalize-batch.pl new file mode 100755 index 00000000..e574f861 --- /dev/null +++ b/corpus/support/utf8-normalize-batch.pl @@ -0,0 +1,28 @@ +#!/usr/bin/env perl + +use IPC::Open2; + +$|++; + +if (scalar(@ARGV) != 1) { + print STDERR "usage: $0 \"CMD\"\n"; + exit(2); +} + +$CMD = $ARGV[0]; + +while (<STDIN>) { + s/\r\n*/\n/g; + $PID = open2(*SOUT, *SIN, $CMD); + print SIN "$_\n"; + close(SIN); + $_ = <SOUT>; + close(SOUT); + waitpid($PID, 0); + chomp; + s/[\x00-\x1F]+/ /g; + s/ +/ /g; + s/^ //; + s/ $//; + print "$_\n"; +} |