From 1bd9f455fc63c40e5dd529a5b7c257d7fd12a40e Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 9 Jun 2021 10:53:39 +0200 Subject: [PATCH] add abbreviations from legacy tokenizer These abbreviations are not a perfect fit anymore because abbreviation replacement is now applied before transliteration. --- settings/legacy_icu_tokenizer.yaml | 807 ++++++++++++++++++++++++++++- 1 file changed, 788 insertions(+), 19 deletions(-) diff --git a/settings/legacy_icu_tokenizer.yaml b/settings/legacy_icu_tokenizer.yaml index 34cd8b0b..84effea4 100644 --- a/settings/legacy_icu_tokenizer.yaml +++ b/settings/legacy_icu_tokenizer.yaml @@ -22,6 +22,7 @@ compound_suffixes: - hütte - kapelle - kogel + - markt - pfad - platz - quelle @@ -84,33 +85,801 @@ compound_suffixes: - plats - platsen abbreviations: - # German + - acceso => acces + - access => accs + - acequia => aceq + - air force base => afb + - air national guard base => angb + - alameda => alam + - ale => al + - alea => al + - aleea => al + - aleja => al + - alejach => al + - aleje => al + - aleji => al + - all => al + - allee => al + - alley => al + - alleyway => alwy + - ally => al + - alqueria => alque + - alue => al + - aly => al - am => a + - amble => ambl - an der => a d - - allgemeines krankenhaus => akh - - altstoffsammelzentrum => asz + - andador => andad + - angosta => angta + - apartamentos => aptos + - apartments => apts + - apch => app + - apeadero => apdro + - approach => app + - arboleda => arb + - arcade => arc + - arrabal => arral + - arroyo => arry + - arterial => artl + - artery => arty - auf der => a d - - bach => b + - aukio => auk + - autopista => auto + - autovia => autov + - avd => av + - avda => av + - ave => av + - avenida => av + - avenue => av + - avinguda => av + - b dul => bd + - back => bk - bad => b - - bahnhof => bhf,bf + - bahnhof => bf + - bajada => bjada + - balneario => balnr + - banan => ba + - banda => b + - barranco => branc + - barranquil => bqllo + - barriada => barda + - barrio => bo + - barro => bo + - basin => basn + - bda => b + - bdge => bri + - bdul => bd + - bdwy => bway + - beach => bch - berg => bg - - bezirk => bez + - bgm => bgm + - bhf => bf + - bldngs => bldgs + - block => blk + - blok => bl + - bloque => blque + - blv => bd + - blvd => bd + - boardwalk => bwlk + - boulevard => bd + - boundary => bdy + - brace => br + - brazal => brzal + - brdg => bri + - break => brk + - bridge => bri + - broadway => bway + - broeder => br - brücke => br + - buildings => bldgs + - bul => bd + - bulev => bd + - bulevar => bd + - bulevard => bd + - bulevardu => bd + - bulevardul => bd + - bulievard => bd + - bulvar => bl + - bulvari => bl - burg => bg - - chaussee => ch - - deutsche,deutscher,deutsches => dt + - burgemeester => bg + - burgermeister => bgm + - business => bus + - buu dien => bd + - bvd => bd + - bypa => byp + - bypass => byp + - byway => bywy + - c le => c + - cadde => cd + - caddesi => cd + - calle => c + - callej => cjon + - calleja => cllja + - callejon => cjon + - callejuela => cjla + - callizo => cllzo + - calzada => czada + - camino => cno + - camino hondo => c h + - camino nuevo => c n + - camino viejo => c v + - camping => campg + - campo => c po + - can cu khong quan => cckq + - cantera => cantr + - cantina => canti + - canton => cant + - cao dang => cd + - caravan => cvn + - carrer => c + - carrera => cra + - carrero => cro + - carretera => ctra + - carreterin => ctrin + - carretil => crtil + - carril => crril + - caserio => csrio + - cau ldhc bo => clb + - caus => cway + - causeway => cway + - ce => cv + - cen => ctr + - center => ctr + - centre => ctr + - centreway => cnwy + - cesta => c + - chalet => chlet + - chase => ch + - che => ch + - chemin => ch + - cinturon => cint + - circle => cir + - circuit => cct + - circunvalacion => ccvcn + - circus => crcs + - city => cty + - cl => c + - cllon => cjon + - close => c + - cmno => cno + - cobertizo => cbtiz + - colonia => col + - commandant => cmdt + - common => comm + - community => comm + - complejo => compj + - concourse => con + - cong truong => ct + - cong ty => cty + - cong ty co phyn => ctcp + - cong vien => cv + - cong vien van hoa => cvvh + - conjunto => cjto + - convento => cnvto + - cooperativa => coop + - copse => cps + - corner => cnr + - corral => crral + - corralillo => crrlo + - corredor => crrdo + - corso => c so + - corte => c te + - cortijo => crtjo + - costanilla => cstan + - costera => coste + - cottages => cotts + - county => co + - county route => cr + - cours => crs + - court => ct + - courtyard => ctyd + - cove => cv + - creek => cr + - cres => cr + - crescent => cr + - crest => crst + - crk => cr + - croft => cft + - cross => crss + - crossing => xing + - crossroad => crd + - crossway => cowy + - crsg => xing + - crt => ct + - csac => cds + - cswy => cway + - cty cp => ctcp + - cuadra => cuadr + - cuesta => custa + - cul de sac => cds + - cutting => cutt + - ddhi hoc => dh + - ddhi lo => dl + - dehesa => dhsa + - demarcacion => demar + - deviation => devn + - diagonal => diag + - diseminado => disem + - distributor => dstr + - doctor => dr + - dokter => dr + - doktor => d r + - dolna => dln + - dolne => dln + - dolny => dln + - dominee => ds - dorf => df - - doktor => dr - - fachhochschule => fh - - Freiwillige Feuerwehr => ff + - dotsient => dots + - drive => dr + - driveway => dvwy + - druga => 2 + - drugi => 2 + - drugie => 2 + - drv => dr + - drwy => dvwy + - duong => d + - duong sat => ds + - duza => dz + - duze => dz + - duzy => dz + - east => e + - edificio => edifc + - elbow => elb + - empresa => empr + - entrada => entd + - entrance => ent + - errepidea => err + - escalera => esca + - escalinata => escal + - espalda => eslda + - esplanade => esp + - estacion => estcn + - estate => est + - estrada => estda + - etorbidea => etorb + - explanada => expla + - expressway => exp + - expy => exp + - extension => ex + - extramuros => extrm + - extrarradio => extrr + - fabrica => fca + - fairway => fawy + - faubourg => fg + - fbrca => fca + - ferry => fy + - fire track => ftrk + - firetrail => fit + - follow => folw + - fondamenta => f ta + - footway => ftwy + - foreshore => fshr + - formation => form + - fort => ft + - freeway => fwy + - front => frnt + - frontage => frtg + - frwy => fwy + - fundacul => fdc + - fundatura => fnd + - ga => g + - galeria => gale + - gamla => gla + - garden => gdn + - gardens => gdn + - gata => g + - gatan => g + - gate => g + - gaten => g + - gdns => gdn + - gebroeders => gebr + - generaal => gen + - gienieral => ghien + - glade => gl + - gld => gl + - glde => gl + - glorieta => gta + - gorna => gn + - gorne => gn + - gorny => gn + - gracht => gr + - grad => ghr + - gran via => g v + - grand => gr + - granden => gr + - grange => gra + - granja => granj + - green => gn + - grn => gn + - gro => gr + - grosse => gr + - grosser => gr + - grosses => gr + - ground => grnd + - grove => gr + - gt => g + - gte => g + - gully => gly + - hauptbahnhof => hbf + - heights => hts + - heiligen => hl + - hgts => hts + - high school => hs + - highroad => hrd + - highway => hwy + - hipodromo => hipod + - hird => hrd + - hospital => hosp + - house => ho + - hse => ho + - i => 1 + - ii => 2 + - iii => 3 + - im => 1 + - impasse => imp + - in => 1 + - in der => i d + - industrial => ind + - ingenieur => ir + - interchange => intg + - international => intl + - intr => int + - intrarea => int + - island => is + - jardin => jdin + - jonkheer => jhr + - junction => jnc + - k s => ks + - kaari => kri + - kalea => k + - kanunnik => kan + - kapitan => kap + - kardinaal => kard + - katu => k + - khach sdhn => ks + - khu cong nghiep => kcn + - khu du lich => kdl + - khu nghi mat => knm + - kleine => kl + - kleiner => kl + - kleines => kl + - kolo => k + - kolonel => kol + - kolonia => kol + - koning => kon + - koningin => kon + - kort e => k + - kri => kri + - kte => k + - kuja => kj + - kvartal => kv + - kyla => kl + - laan => ln + - ladera => ldera + - landing => ldg + - lane => ln + - laneway => lnwy + - lange => l + - largo => l go + - lille => ll + - line => ln + - link => lk + - lit => lt + - little => lt + - llanura => llnra + - lookout => lkt + - loop => lp + - low => lr + - lower => lr + - luitenant => luit + - lwr => lr + - m te => mt + - m tele => mt + - maantee => mt + - mala => ml + - male => ml + - malecon => malec + - maly => ml + - manor => mnr + - mansions => mans + - market => mkt + - markt => mkt + - mazowiecka => maz + - mazowiecki => maz + - mazowieckie => maz + - meadows => mdws + - meander => mr + - medical => med + - meester => mr + - mercado => merc + - mevrouw => mevr + - mews => m + - miasto => m + - middle => mid + - middle school => ms + - mile => mi + - military => mil + - mirador => mrdor + - mitropolit => mit + - mkt => mkt + - mndr => mr + - mnt => mt + - monasterio => mtrio + - monseigneur => mgr + - mont => mt + - motorway => mwy + - mount => mt + - mountain => mtn + - mtwy => mwy + - muelle => muell + - municipal => mun + - muntele => mt + - museum => mus + - mw => m + - na => n + - namesti => nam + - namestie => nam + - national park => np + - national recreation area => nra + - national wildlife refuge area => nwra + - nha hat => nh + - nha thi dzu => ntd + - nha tho => nt + - nordre => ndr + - norra => n + - north => n + - north east => ne + - north west => n + - northeast => ne + - northwest => n + - nowa => n + - nowe => n + - nowy => n + - nucleo => ncleo + - nw => n + - oa => o + - ob => o + - obere => o + - oberer => o + - oberes => o + - olv => olv + - onze lieve vrouw e => olv + - osiedle => os + - osiedlu => os + - ostra => o + - outlook => otlk + - p k => pk + - p ta => pta + - p zza => p za + - palacio => palac + - pantano => pant + - parade => pde + - paraje => praje + - park => pk + - parklands => pkld + - parkway => pwy + - parque => pque + - particular => parti + - partida => ptda + - pas => ps + - pasadizo => pzo + - pasaje => psaje + - paseo => po + - paseo maritimo => psmar + - pasillo => psllo + - pass => ps + - passage => ps + - passatge => ptge + - passeig => pg + - pastoor => past + - pathway => pway + - penger => pgr + - pfad => p + - pgr => pgr + - ph => p + - phi truong => pt + - phuong => p + - phwy => pway + - piata => pta + - piaz => p za + - piazza => p za + - piazzale => p le + - piazzetta => pta + - pierwsza => 1 + - pierwsze => 1 + - pierwszy => 1 + - pike => pk + - pko => pko + - pkwy => pwy + - pky => pwy + - plac => pl + - placa => pl + - place => pl + - placem => pl + - placu => pl + - plass => pl + - plassen => pl + - plateau => plat + - plats => pl + - platsen => pl + - platz => pl + - plaza => pl + - plazoleta => pzta + - plazuela => plzla + - plein => pln + - pln => pln + - ploshchad => pl + - plz => pl + - plza => pl + - pnt => pt + - poblado => pbdo + - pocket => pkt + - point => pt + - poligono => polig + - poligono industrial => pgind + - polku => p + - ponte => p te + - porta => pta + - portal => prtal + - portico => prtco + - portillo => ptilo + - prazuela => przla + - precinct => pct + - pres => pres + - president => pres + - prins => pr + - prinses => pr + - professor => prof + - profiesor => prof + - prolongacion => prol + - promenade => prom + - psge => ps + - pueblo => pblo + - puente => pnte + - puerta => pta + - puerto => pto + - puistikko => pko + - puisto => ps + - punto kilometrico => pk + - pza => pl + - quadrangle => qdgl + - quadrant => qdrt + - quai => qu + - quan => q + - quay => qy + - quays => qys + - qucyng truong => qt + - quelle => qu + - quoc lo => ql + - raitti => r + - rambla => rbla + - ramble => rmbl + - rampla => rampa + - range => rnge + - ranta => rt + - rdhp hat => rh + - reach => rch + - reservation => res + - reserve => res + - reservoir => res + - residencial => resid + - retreat => rtt + - rhein => rh + - ribera => rbra + - ridge => rdge + - ridgeway => rgwy + - right of way => rowy + - rincon => rcon + - rinconada => rcda + - rinne => rn + - rise => ri + - riv => r + - river => r + - riverway => rvwy + - riviera => rvra + - road => rd + - roads => rds + - roadside => rdsd + - roadway => rdwy + - ronde => rnde + - rosebowl => rsbl + - rotary => rty + - rotonda => rtda + - round => rnd + - route => rt + - rte => rt + - rue => r + - rvr => r + - sa => s + - saint => st + - sainte => ste + - salizada => s da + - san => s + - san bay => sb + - san bay quoc te => sbqt + - san van dong => svd + - sanatorio => sanat - sankt => st + - santa => sta + - santuario => santu + - sector => sect + - sendera => sedra + - sendero => send + - serviceway => swy + - shunt => shun + - shwy => sh + - siding => sdng + - sielo => s + - sint => st + - slope => slpe + - sodra => s + - sok => sk + - sokagi => sk + - sokak => sk + - sondre => sdr + - soseaua => sos + - sound => snd + - south => s + - south east => se + - south west => sw + - south-east => se + - south-west => sw + - southeast => se + - southwest => sw + - spl => sp + - splaiul => sp + - spodnja => sp + - spodnje => sp + - spodnji => sp + - square => sq + - srednja => sr + - srednje => sr + - srednji => sr + - stara => st + - stare => st + - stary => st + - state highway => sh + - state route => sr + - station => stn + - stazione => staz + - ste => ste + - steenweg => stwg + - sth => s + - stig => st + - stigen => st + - store => st + - str la => sdla + - stra => st + - straat => st + - strada => st + - strada comunale => sc + - strada provinciale => sp + - strada regionale => sr + - strada statale => ss + - stradela => sdla + - strand => st - strasse => str - - weg => wg - # English - - alley => al - - beach => bch - street => st - - road => rd - - bridge => brdg - - + - strip => strp + - stwg => stwg + - subida => sbida + - subway => sbwy + - sveta => sv + - sveti => sv + - svieti => sv + - sving => sv + - svingen => sv + - svwy => swy + - taival => tvl + - tanav => tn + - tce => ter + - tcty => tct + - terr => ter + - terrace => ter + - thanh pho => tp + - thfr => thor + - thi trzn => tt + - thi xa => tx + - thoroughfare => thor + - tie => t + - tieu hoc => th + - tinh lo => tl + - tollway => tlwy + - tong cong ty => tct + - tori => tr + - torrente => trrnt + - towers => twrs + - township => twp + - tpke => tpk + - track => trk + - trail => trl + - trailer => trlr + - transito => trans + - transversal => trval + - trasera => tras + - travesia => trva + - triangle => tri + - trung hoc co so => thcs + - trung hoc pho thong => thpt + - trung tam => tt + - trung tam thuong mdhi => tttm + - trunkway => tkwy + - trzeci => 3 + - trzecia => 3 + - trzecie => 3 + - tunnel => tun + - turn => tn + - turnpike => tpk + - tvl => tvl + - ulica => ul + - ulice => ul + - ulicy => ul + - ulitsa => ul + - underpass => upas + - university => univ + - untere => u + - unterer => u + - unteres => u + - upper => up + - upr => up + - urbanizacion => urb + - utca => u + - v d => vd + - va => v + - vag => v + - vagen => v + - vale => v + - van => v + - van de => vd + - varf => vf + - varful => vf + - vastra => v + - vayla => vla + - vdct => via + - vecindario => vecin + - vei => v + - veien => v + - velika => v + - velike => v + - veliki => v + - veliko => v + - vereda => vreda + - via => v + - viad => via + - viaduct => via + - viaducto => vcto + - viale => v le + - vicolo => v lo + - vien bcyo tang => vbt + - view => vw + - villas => vlls + - virf => vf + - virful => vf + - vista => vsta + - viviendas => vvdas + - vkhod => vkh + - vla => vla + - vliet => vlt + - vlt => vlt + - vn => v + - vuon quoc gia => vqg + - walk => wlk + - walkway => wkwy + - way => wy + - west => w + - wharf => whrf + - wielka => wlk + - wielki => wlk + - wielkie => wlk + - wielkopolska => wlkp + - wielkopolski => wlkp + - wielkopolskie => wlkp + - wojewodztwie => woj + - wojewodztwo => woj + - yard => yd + - zgornja => zg + - zgornje => zg + - zgornji => zg + - zhilishchien komplieks => zh k + - zum => z -- 2.39.5