#!/usr/bin/env perl
use warnings;
use utf8;
BEGIN { $| = 1; }
use DB_File;
use DBM_Filter;
use Encode;

while (<>) {

  s/ (\p{Ll}\p{Ll}) \{nill\} \1 / $1 $1 /g;
  s/ (\p{Ll}\p{Ll}) \1 \1 \{nill\}([^\p{L}])/ $1 $1 $1$2/g;
  
s/^/ /; # initial space added for reg.ex. matching, to be removed at the end
s/ +\././g; # addbreak + addfejl may split of dot, e.g. "Der findes pt. ingen ..."

s/\{([^ ]+=) ([^ ]+)\}/{$1$2}/g;
s/\{([^ ]+-)([0-9]+[wßðf]+ )(\pL+[,\}])/{$2$1$3/g; # TV2-vært
while (s/\{([^,\}]*?,)([^ ]+-)([0-9]+[wßðf]+ )(\pL+[,\}])/{$1$3$2$4/g) {} # TV2-vært

s/ (\p{L}+\.)(\p{L}+) \{\.=\2\}/ $1$2 {$1=$2}/g;
s/ (\p{L}+-[0-9]+) \{[^\}-]+\}/ $1/g; # euro-11 {60mßwf euro,58w euros} 

### internet

    s/ (\p{Ll}\p{Ll}\p{Ll}+\.) +\{[0-9]+[mßwfðr]* [^\{]*?} ((com|org|edu|net|[b-z][a-z^å])[^\pL])/ $1$2/g; # jubii.dk ... should have been blocked in addfejl2 already, to avoid mistakes a la 'word. de ...'
#print "--$_\n";

### correction = original

#s/ (\pL+) \{\1\}/ $1/g; # indflydelsessfærer --> not necessary

### wrong splits

s/( \{[0-9]+)=(er\})/$1'$2/; # 70er

### latin MWEs

  s/ (al?|codex|ex|in|pro) \{[0-9]+[mwfsßðr]* [^\{\}]*?\} ([a-z=]*[aiou]s?|[au]m|[ei]s|dente|nihil|ore|res|spe)/ $1 $2/g;
  s/ (al?|codex|ex|in|pro) ([a-z=]*[aiou]s?|[au]m|[ei]s|dente|nihil|ore|res|spe) \{[0-9]+[mwfsßðr]* [^\{\}]*?\}/ $1 $2/g;

### foreign in quotes

if (/^ *\".*\"( ?{[^\}]*} *)?$/) {
  $ct_stav = () = $_ =~ /\{/g; # counts the number of regex matches
  $xx =$_; $xx =~ s/ \{.*?\}//g;
  $ct_words = () = $xx =~ / +/g;
  if ($ct_stav > ($ct_words - 1)/2) {
    $_ = $xx;
    s/ /=/g; s/^=+//; s/=+$//;
    print;
    goto loopend;
  }
#  print "-----ct_words=$ct_words ct_stav=$ct_stav\n";
}

### English ADJ + N expressions

  s/ (big|black|blue|common|free|golden|google|happy|high|little|low|new|pick|public|slow|white) \{[0-9]+[mwfsßðr]* [^\{\}]*?\} (\pL)/ $1 $2/g;
  s/ (big|black|blue|common|free|golden|google|happy|high|little|low|new|pick|public|slow|white) (\pL\p{Ll}+) \{[0-9]+[mwfsßðr]* [^\{\}]*?\}/ $1 $2/g;

### English MWE

  s/ (need) \{[0-9]+[mwfsßðr]* [^\{\}]*?\} (to know)/ $1 $2/g;
  s/ (puff) \{[0-9]+[mwfsßðr]* [^\{\}]*?\} (bar)/ $1 $2/g;

### French N + ADJ expressions

  s/ (\pL\p{Ll}+) \{[0-9]+[mwfsßðr]* [^\{\}]*?\} (blanch?e?|bleu|grise?|noir|rouge|verte)/ $1 $2/g; # e.g. wines: pino blanc
  s/ (\pL\p{Ll}+) (blanch?e?|bleu|grise?|noir|rouge|verte) \{[0-9]+[mwfsßðr]* [^\{\}]*?\}/ $1 $2/g;


### hyphenation problems
  
  s/ (\p{Ll}+)-(\p{Ll}+)-(\p{Ll}+) \{[0-9]+[mwfsßðr]* \1\2-\3\}/ $1-$2-$3/g; # rød-grøn-hvide --> rødgrøn-hvide
  s/(\p{Ll}+)= \{[0-9]+[mwfsßðr]* [^\{\}]*?\}/$1 =/g; # de= det=

  
### units

s/ (([fpmcdhkMGT]?[gmlBVW]|eV|gCO2|mEq|[MGT][Bb]it)[23²³]?(\/([fpmcdhkMGT]?[hmls]|sec|min|t|[a-z]+\.|L)[23²³]?)?) \{[0-9]+[mwfsßðr]* [^\{\}]*?\}/ $1/g;

### abbreviations

s/ ((\pL+\.)+\pL\.) \{[0-9]+[mwfsßðr]* [^{}]*?\}/ $1/g; # good 2-part abbreviation

s/ (\pL+)\. \{[0-9]+[mwfsßðr]*( [^{}]*?)? \1\pL*(,[^\{\}]*?)?\}/ $1. /g; # one-dot abbreviation candidate is not an abbreaviation, if the non-dot part is given as (first part of) a correction suggestion: ... bruger så meget tid det. i dag behøver ... --> leave the decision to dan.pre (or force it by inserting a space before the dot?)

  s/ (\pL)(\pL+)( \{([0-9]+[mwfsßðr]*( [^{}]*?)? )?\1\.\2\.(,[^\{\}]*?)?\}) ?\. */ $1$2. $3/g; # feks. --> f.eks.
  s/ (\pL+)(\pL)( \{([0-9]+[mwfsßðr]*( [^{}]*?)? )?\1\.\2\.(,[^\{\}]*?)?\}) ?\. */ $1$2. $3/g; # bla. --> bl.a.
  s/ (\pL+).: \{([0-9]+[mwfsßðr]*( [^{}]*?)? )?\1(,[^\{\}]*?)?\}/ $1 . :/g; # de daglige rutiner.:
  
s/ (.+)=PPP\. \{\1\\?\.\}/ $1./g; # for ca=PPP. {ca\.} et år siden
s/så=PPYlidt/så lidt/g;
s/(Norden|Vesten|Østen)=PPYom /$1 om /g;
s/=PPP(i) / $1 /g; # privatøkonom=PPYi Arbejdernes Landsbank
  
#s/ ([b-df-hj-np-tvxz]+) {[0-9]+[mwfsßðr]* [^{}]*?} \. (\p{Ll})/ $1\. $2/g; # risky and unnecessary since addfejl.subs2 opts out in this case (search "det vejer 4 pns. og er grøn")
  s/ (bla) \{[0-9]+[mwfsßðr]* [^{}]*?\} ?\. (\p{L})/ $1\. $2/g; # wrong abbreviations: 'bla.' (wouldn't show up in fuldformer_stav that only contains correct forms, hence not part of freq)
  
s/ ([B-DF-HJ-NP-TVXZ][b-df-hj-np-tvxz]+|S[ck]t)\. \{[0-9]+[mwfsßðr]* [^\{\}]*?\} (\p{Lu})/ $1\. $2/g; # St. Thomas Dock, Mt. Cook
s/ ([B-DF-HJ-NP-TVXZ][b-df-hj-np-tvxz]+|S[ck]t) \{[0-9]+[mwfsßðr]* [^\{\}]*?\}\. (\p{Lu})/ $1\. $2/g; # Sct. Olaigade

#print "--$_\n";

s/ (\pL+) \{\1\.}\.\./ $1. ./g; # 400 mio. kr..

### names

  s/ ([\'\"]?[A-ZÆØÅ]\w+(-\p{Lu}\pL*)*|bin|di|d[ao]s|ibn|in|les?|v[ao]n|I['’]m) \{[0-9]+[mwfsßðr]* [^{}]*?\} ((& )?[A-ZÆØÅ])/ $1 $3/g; # undo addfejl's non-prop suggestions for first prop part: Ståle Solbakken
s/ (\p{Lu}\p{Ll}+) (\p{Lu}\p{Ll}+)'s \{[0-9]+[mwfsßðr]* [^{}]*?\} / $1 $2's /g; # den indonesiske musikfestival We The Fest's instagram-profil
  
  s/ ([\'\"]?[vV]an) \{[0-9]+[mwfsßðr]* [^\{\}]*?\} (de[nr] [A-ZÆØÅ])/ $1 $2/g; # van der Doel
  s/ (\p{Lu}\p{Ll}+) \{[0-9]+[mwfsßðr]* [^\{\}]*?\} (al) \{.*?\} (\p{Lu}\p{Ll}+) / $1 $2 $3/g; # Burj al Arab/Kalifa/Babas
  s/ ((Nr|Sdr)\. \p{Lu}\p{Ll}+) \{[0-9]+[mwfsßðr]* [^\{\}]*?\}/ $1 /g; # Nr. Aslev

s/ ([\'\"]?[A-ZÆØÅ]\w+(-\p{Lu}\pL*)*) \{[0-9]+[mwfsßðr]* [^\{\}]*?\} ((al-)?\p{Lu}\w*(-\pL+)*|bin|di|d[ao]s|ibn|in|les?|v[ao]n) / $1 $3 /g; # undo addfejl's non-prop suggestions for first prop part: Ståle Solbakken

s/ (Kommite) (\{[0-9]+[mwfsßðr]*) ([a-zæøå])/ $1  $2 \u$3/g; # protect against removal of corr.suggestions for 2nd part of PROP
  s/ ([\'\"]?(\p{Lu}\pL*(-\p{Lu}\pL*)*|&|bin|di|d[ao]s|ibn|in|les?|v[ao]n|van de[nr]|(\p{Lu}|hr|frk?|Mrs?)\.) (\p{Lu}\pL*(-\pL+)*|al-\p{Lu}\w*(-\pL+)*)[\'\"]?) \{[0-9]+[mwfsßðr]* [^\{\}]*?\}/ $1 /g; # don't correct second part in PROP chain either ... but problem: Ruslands Statslige Statistiske Kommite (correct: Komite)
    s/(Stationen).(Co) \{.*?\}/$1\*xdot$2 /g;

    s/ (\p{Lu}\p{Ll}+) (\p{Lu}\p{Ll}+)=PPP(\p{Ll}+) \{[^\}]*\}/ $1 $2 $3/g; # Det var netop Joy Me der skabte overskrifter

#  print "--$_\n";
  s/([^ ]) +([^ ]+) +\{/$1 xywhitespace $2 {/g; # block dan.pre's possible attempt at = fusions left of addfejl suggestions, since those will only concern the last word. E.g. title candidates in quotes: "En rigtig Parkervin", siger Gert Carl ... but only break line if there is text to the left - at line start this would lead to an empty line and ultimately "<¶>" ... problem: "<¶>" is also inserted if both the word before and after is upper case: "På LærIT's hjemmeside ligger der ..." --> use xywhitespace with surrounding spaces instead (without surrounding spaces it will mess up the *-prefixing of upper case tokens)
s/([^ ]) xywhitespace ([^ ]+)\. \{[^ ]+ \2\}( *)$/$1 $2\.$3/g;
s/([^ ]) xywhitespace ([^ ]+)\. \{[^ ]+ \2\}([^\p{L} ]| [^\p{Ll}])/$1 $2\.$3/g;
s/([^ ]') xywhitespace (erne|eren) \{.*?\}/$1 $2/g; # 40' erne, F35' eren
s/' ?([\pL ]*) xywhitespace ([^ ]+'-\pL+) \{.*?\}/'$1 $2/g; # på bedste ' Kejserens nye klæder'-vis
s/( xywhitespace [^ ]+)-\. /$1\*xhyfen\*xdot /g;
s/^( *'[^']+)'( \{.*?\})/$1$2'/g;
s/^( *"[^"]+)"( \{.*?\})/$1$2"/g;
s/ xywhitespace (bint) \{.*\}/ $1/g;
s/([ =]\p{Lu}\p{L}+) (bint?|des?|d[ui]|d[ao]s|ibn|v[ao]n) xywhitespace (\p{Lu}\p{L}+) \{.*?\}/$1=$2=$3/g;
s/([ =]\p{Lu}\p{L}+) (bint?|des?|d[ui]|d[ao]s|ibn|v[ao]n) xywhitespace (\p{Lu}\p{L}+) \{.*?\}/$1=$2=$3/g;
s/ xywhitespace( \p{Lu}\p{L}+) \{.*?\} (bint?|des?|d[ui]|d[ao]s|ibn|v[ao]n) ((al-)?\p{Lu})/$1=$2=$3/;
s/ xywhitespace (\p{L}\p{Ll}+)\.(\p{Lu}\p{Ll}*) \{.*?[^\.]\}/ $1.\n<sentsplit>\n$2/g; # det giver xywhitespace ungdomsopør.Det {74mwf ungdomsoprøret,69ß ungdomspartiet,61 ungdomsoprørets,58 ungdomsoprøreren,58 ungdomsoprøreres} er klart
s/(\p{L})\.(\p{L}+) \{\.=\1\}/$1.\n<sentsplit>\n$2/g; # det giver bagslag.Forskningsadjunkt {.=Forskningsadjunkt} Meyer kan se det.
  s/ (\p{L}+)\.(\p{Lu}+\p{L}*) ([^\{])/ $1.\n<sentsplit>\n$2 $3/g; # xxx.Yyy cases not recognized by addfejl2 (xxx <=4 letters, Yyy <=3 letters and not on list?), e.g. ' støj.I '
#  print "--$_***\n";
s/^( *(\p{Lu}+ )+)(\p{Lu}+) (\p{Lu}\p{Ll})/$1$3\n<sentsplit>\n$4/g;
s/ xywhitespace (haft)(\p{Ll}+) \{[^\}=]*\}/ $1$2 {$1=$2}/g;
s/ xywhitespace (\p{L}+e)=PPPde \{\1de\}/ $1 de/g;
s/ xywhitespace (\p{L}+)\*xdot \{.*\}" +(\p{Lu})/ $1 ."\n$2/g;
s/ xywhitespace (\p{L}+)\*xdot \{.*\}" *$/ $1 ."/g; # "overraskende xywhitespace gennemført*xdot {58mßwf gennemført}"
  s/(-[0-9]+\.) xywhitespace/$1/g; # det konkrete formål i 2020-2023. xywhitespace Vær {100ð vær,100ð være,40ð værd,40ð vejr,40ð hver} opmærksom på
  
#print "--$_\n";

### animals, bacteria, plants etc.

    s/([A-Z]([a-z]+ )+ \{[0-9]+[mßwfðr]* [^{]*?\} (caudalis|cranialis|dexter|distalis|inferior|lateralis|maximus|medialis|minor|maior|proximalis|radialis|sinister|superior|ulnaris|[a-z]+oides))([^\w])/§$1¤$4/; # Ursa maior, Vinca minor, Flexor carpi ulnaris

    s/([A-Z][a-z][a-z]+(ea|era|is|ic?a|um|us)|[A-Z]\.) +\{[0-9]+[mßwfðr]* [^{]*?\} ([a-z][a-z][a-z]+([ae]ns|ea|era|[cdlmnrs]is|i|ic?a|um|us)|[a-z]\.) +([a-z][a-z][a-z]+([ae]ns|ea|era|[cdlmnrs]is|i|ic?a|um|us)[^a-z])/$1=$3=$5/g; # C. sinensis assamica, Clostridium perfringens
    s/([A-Z][a-z][a-z]+(ea|era|is|ic?a|um|us)|[A-Z]\*xdot|[A-Z]\.[a-z]\.) +{[0-9]+[mßwfðr]* [^{]*?} ([a-z][a-z][a-z]+([ae]ns|[eiou]a|era|[cdlmnrs]is|i|ic?a|osa|um|us)[^a-z])/$1=$3/g; # Bacillus subtilis, G. sonneratii, Felis tigris, Eugoa pulverosa, G.c. reticulata

s/hvad det=PPYvil=PPYsige/hvad det vil sige/g; # correct over-fusion

s/^ +//; # remove the initial space(s) added in the beginning, which might otherwise confuse dan.pre (dan.pre only reckons with its own 1 added line-start space)

  print;
 loopend:
}

