corpus/tokenize-anything.sh


1
2
3
4
5
6
7
8
9
10
11
12
13

#!/bin/sh

ROOTDIR=`dirname $0`
SUPPORT=$ROOTDIR/support

$SUPPORT/utf8-normalize.sh |
  $SUPPORT/quote-norm.pl |
  $SUPPORT/tokenizer.pl |
  sed -e 's/ al - / al-/g' |
  $SUPPORT/fix-contract.pl |
  sed -e 's/^ //' | sed -e 's/ $//' |
  perl -e 'while(<>){s/(\d+)(\.+)$/$1 ./;print;}'