summaryrefslogtreecommitdiff
path: root/corpus/tokenize-anything.sh
blob: c7adfa611ac42aad34e2d66cd3c5a3b385d3d8ad (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
#!/usr/bin/env bash

ROOTDIR=`dirname $0`
SUPPORT=$ROOTDIR/support

if [[ $# == 1 && $1 == '-u' ]] ; then
    NORMARGS="--batchline"
else
    NORMARGS=""
fi

$SUPPORT/utf8-normalize.sh $NORMARGS |
  $SUPPORT/quote-norm.pl |
  $SUPPORT/tokenizer.pl |
  sed -e 's/ al - / al-/g' |
  $SUPPORT/fix-contract.pl |
  sed -e 's/^ //' | sed -e 's/ $//' |
  perl -e '$|++; while(<>){s/(\d+)(\.+)$/$1 ./; s/(\d+)(\.+) \|\|\|/$1 . |||/;  print;}'