translate.filters.checks

1 #!/usr/bin/env python 2 # -*- coding: utf-8 -*- 3 # 4 # Copyright 2004-2008 Zuza Software Foundation 5 # 6 # This file is part of translate. 7 # 8 # translate is free software; you can redistribute it and/or modify 9 # it under the terms of the GNU General Public License as published by 10 # the Free Software Foundation; either version 2 of the License, or 11 # (at your option) any later version. 12 # 13 # translate is distributed in the hope that it will be useful, 14 # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 # GNU General Public License for more details. 17 # 18 # You should have received a copy of the GNU General Public License 19 # along with translate; if not, write to the Free Software 20 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 21 22 """This is a set of validation checks that can be performed on translation 23 units. 24 25 Derivatives of UnitChecker (like StandardUnitChecker) check translation units, 26 and derivatives of TranslationChecker (like StandardChecker) check 27 (source, target) translation pairs. 28 29 When adding a new test here, please document and explain the behaviour on the 30 U{wiki <http://translate.sourceforge.net/wiki/toolkit/pofilter_tests>}. 31 """ 32 33 import re 34 35 from translate.filters import helpers 36 from translate.filters import decoration 37 from translate.filters import prefilters 38 from translate.filters import spelling 39 from translate.lang import factory 40 from translate.lang import data 41 # The import of xliff could fail if the user doesn't have lxml installed. For 42 # now we try to continue gracefully to help users who aren't interested in 43 # support for XLIFF or other XML formats. 44 try: 45 from translate.storage import xliff 46 except ImportError, e: 47 xliff = None 48 # The import of xliff fail silently in the absence of lxml if another module 49 # already tried to import it unsuccessfully, so let's make 100% sure: 50 if not hasattr(xliff, "xliffunit"): 51 xliff = None 52 53 # These are some regular expressions that are compiled for use in some tests 54 55 # printf syntax based on http://en.wikipedia.org/wiki/Printf which doens't 56 # cover everything we leave \w instead of specifying the exact letters as 57 # this should capture printf types defined in other platforms. 58 # extended to support Python named format specifiers 59 printf_pat = re.compile('%((?:(?P<ord>\d+)\$|$(?P<key>\w+)$)?(?P<fullvar>[+#-]*(?:\d+)?(?:\.\d+)?(hh\|h\|l\|ll)?(?P<type>[\w%])))') 60 61 # The name of the XML tag 62 tagname_re = re.compile("<[\s]*([\w\/]*)") 63 64 # We allow escaped quotes, probably for old escaping style of OOo helpcontent 65 #TODO: remove escaped strings once usage is audited 66 property_re = re.compile(" (\w*)=((\\\\?\".*?\\\\?\")|(\\\\?'.*?\\\\?'))") 67 68 # The whole tag 69 tag_re = re.compile("<[^>]+>") 70 71 gconf_attribute_re = re.compile('"[a-z_]+?"') 72 73

74 -def tagname(string):

75 """Returns the name of the XML/HTML tag in string""" 76 return tagname_re.match(string).groups(1)[0]

77 78

79 -def intuplelist(pair, list):

80 """Tests to see if pair == (a,b,c) is in list, but handles None entries in 81 list as wildcards (only allowed in positions "a" and "c"). We take a 82 shortcut by only considering "c" if "b" has already matched.""" 83 a, b, c = pair 84 if (b, c) == (None, None): 85 #This is a tagname 86 return pair 87 for pattern in list: 88 x, y, z = pattern 89 if (x, y) in [(a, b), (None, b)]: 90 if z in [None, c]: 91 return pattern 92 return pair

93 94

95 -def tagproperties(strings, ignore):

96 """Returns all the properties in the XML/HTML tag string as 97 (tagname, propertyname, propertyvalue), but ignore those combinations 98 specified in ignore.""" 99 properties = [] 100 for string in strings: 101 tag = tagname(string) 102 properties += [(tag, None, None)] 103 #Now we isolate the attribute pairs. 104 pairs = property_re.findall(string) 105 for property, value, a, b in pairs: 106 #Strip the quotes: 107 value = value[1:-1] 108 109 canignore = False 110 if (tag, property, value) in ignore or \ 111 intuplelist((tag, property, value), ignore) != (tag, property, value): 112 canignore = True 113 break 114 if not canignore: 115 properties += [(tag, property, value)] 116 return properties

117 118

119 -class FilterFailure(Exception):

120 """This exception signals that a Filter didn't pass, and gives an 121 explanation or a comment""" 122

123 - def __init__(self, messages):

124 if not isinstance(messages, list): 125 messages = [messages] 126 assert isinstance(messages[0], unicode) # Assumption: all of same type 127 joined = u", ".join(messages) 128 Exception.__init__(self, joined) 129 # Python 2.3 doesn't have .args 130 if not hasattr(self, "args"): 131 self.args = joined

132 133

134 -class SeriousFilterFailure(FilterFailure):

135 """This exception signals that a Filter didn't pass, and the bad translation 136 might break an application (so the string will be marked fuzzy)""" 137 pass

138 139 #(tag, attribute, value) specifies a certain attribute which can be changed/ 140 #ignored if it exists inside tag. In the case where there is a third element 141 #in the tuple, it indicates a property value that can be ignored if present 142 #(like defaults, for example) 143 #If a certain item is None, it indicates that it is relevant for all values of 144 #the property/tag that is specified as None. A non-None value of "value" 145 #indicates that the value of the attribute must be taken into account. 146 common_ignoretags = [(None, "xml-lang", None)] 147 common_canchangetags = [("img", "alt", None), 148 (None, "title", None), 149 (None, "dir", None), 150 (None, "lang", None), 151 ] 152 # Actually the title tag is allowed on many tags in HTML (but probably not all) 153 154

155 -class CheckerConfig(object):

156 """object representing the configuration of a checker""" 157

158 - def __init__(self, targetlanguage=None, accelmarkers=None, varmatches=None, 159 notranslatewords=None, musttranslatewords=None, 160 validchars=None, punctuation=None, endpunctuation=None, 161 ignoretags=None, canchangetags=None, criticaltests=None, 162 credit_sources=None):

163 # Init lists 164 self.accelmarkers = self._init_list(accelmarkers) 165 self.varmatches = self._init_list(varmatches) 166 self.criticaltests = self._init_list(criticaltests) 167 self.credit_sources = self._init_list(credit_sources) 168 # Lang data 169 self.targetlanguage = targetlanguage 170 self.updatetargetlanguage(targetlanguage) 171 self.sourcelang = factory.getlanguage('en') 172 # Inits with default values 173 self.punctuation = self._init_default(data.normalized_unicode(punctuation), 174 self.lang.punctuation) 175 self.endpunctuation = self._init_default(data.normalized_unicode(endpunctuation), 176 self.lang.sentenceend) 177 self.ignoretags = self._init_default(ignoretags, common_ignoretags) 178 self.canchangetags = self._init_default(canchangetags, common_canchangetags) 179 # Other data 180 # TODO: allow user configuration of untranslatable words 181 self.notranslatewords = dict.fromkeys([data.normalized_unicode(key) for key in self._init_list(notranslatewords)]) 182 self.musttranslatewords = dict.fromkeys([data.normalized_unicode(key) for key in self._init_list(musttranslatewords)]) 183 validchars = data.normalized_unicode(validchars) 184 self.validcharsmap = {} 185 self.updatevalidchars(validchars)

186

187 - def _init_list(self, list):

188 """initialise configuration paramaters that are lists 189 190 @type list: List 191 @param list: None (we'll initialise a blank list) or a list paramater 192 @rtype: List 193 """ 194 if list is None: 195 list = [] 196 return list

197

198 - def _init_default(self, param, default):

199 """initialise parameters that can have default options 200 201 @param param: the user supplied paramater value 202 @param default: default values when param is not specified 203 @return: the paramater as specified by the user of the default settings 204 """ 205 if param is None: 206 return default 207 return param

208

209 - def update(self, otherconfig):

210 """combines the info in otherconfig into this config object""" 211 self.targetlanguage = otherconfig.targetlanguage or self.targetlanguage 212 self.updatetargetlanguage(self.targetlanguage) 213 self.accelmarkers.extend([c for c in otherconfig.accelmarkers if not c in self.accelmarkers]) 214 self.varmatches.extend(otherconfig.varmatches) 215 self.notranslatewords.update(otherconfig.notranslatewords) 216 self.musttranslatewords.update(otherconfig.musttranslatewords) 217 self.validcharsmap.update(otherconfig.validcharsmap) 218 self.punctuation += otherconfig.punctuation 219 self.endpunctuation += otherconfig.endpunctuation 220 #TODO: consider also updating in the following cases: 221 self.ignoretags = otherconfig.ignoretags 222 self.canchangetags = otherconfig.canchangetags 223 self.criticaltests.extend(otherconfig.criticaltests) 224 self.credit_sources = otherconfig.credit_sources

225

226 - def updatevalidchars(self, validchars):

227 """updates the map that eliminates valid characters""" 228 if validchars is None: 229 return True 230 validcharsmap = dict([(ord(validchar), None) for validchar in data.normalized_unicode(validchars)]) 231 self.validcharsmap.update(validcharsmap)

232

233 - def updatetargetlanguage(self, langcode):

234 """Updates the target language in the config to the given target 235 language""" 236 self.lang = factory.getlanguage(langcode)

237 238

239 -def cache_results(f):

240 241 def cached_f(self, param1): 242 key = (f.__name__, param1) 243 res_cache = self.results_cache 244 if key in res_cache: 245 return res_cache[key] 246 else: 247 value = f(self, param1) 248 res_cache[key] = value 249 return value

250 return cached_f 251 252

253 -class UnitChecker(object):

254 """Parent Checker class which does the checking based on functions available 255 in derived classes.""" 256 preconditions = {} 257

258 - def __init__(self, checkerconfig=None, excludefilters=None, 259 limitfilters=None, errorhandler=None):

260 self.errorhandler = errorhandler 261 if checkerconfig is None: 262 self.setconfig(CheckerConfig()) 263 else: 264 self.setconfig(checkerconfig) 265 # exclude functions defined in UnitChecker from being treated as tests. 266 self.helperfunctions = {} 267 for functionname in dir(UnitChecker): 268 function = getattr(self, functionname) 269 if callable(function): 270 self.helperfunctions[functionname] = function 271 self.defaultfilters = self.getfilters(excludefilters, limitfilters) 272 self.results_cache = {}

273

274 - def getfilters(self, excludefilters=None, limitfilters=None):

275 """returns dictionary of available filters, including/excluding those in 276 the given lists""" 277 filters = {} 278 if limitfilters is None: 279 # use everything available unless instructed 280 limitfilters = dir(self) 281 if excludefilters is None: 282 excludefilters = {} 283 for functionname in limitfilters: 284 if functionname in excludefilters: 285 continue 286 if functionname in self.helperfunctions: 287 continue 288 if functionname == "errorhandler": 289 continue 290 filterfunction = getattr(self, functionname, None) 291 if not callable(filterfunction): 292 continue 293 filters[functionname] = filterfunction 294 return filters

295

296 - def setconfig(self, config):

297 """sets the accelerator list""" 298 self.config = config 299 self.accfilters = [prefilters.filteraccelerators(accelmarker) for accelmarker in self.config.accelmarkers] 300 self.varfilters = [prefilters.filtervariables(startmatch, endmatch, prefilters.varname) 301 for startmatch, endmatch in self.config.varmatches] 302 self.removevarfilter = [prefilters.filtervariables(startmatch, endmatch, 303 prefilters.varnone) 304 for startmatch, endmatch in self.config.varmatches]

305

306 - def setsuggestionstore(self, store):

307 """Sets the filename that a checker should use for evaluating 308 suggestions.""" 309 self.suggestion_store = store 310 if self.suggestion_store: 311 self.suggestion_store.require_index()

312

313 - def filtervariables(self, str1):

314 """filter out variables from str1""" 315 return helpers.multifilter(str1, self.varfilters)

316 filtervariables = cache_results(filtervariables) 317

318 - def removevariables(self, str1):

319 """remove variables from str1""" 320 return helpers.multifilter(str1, self.removevarfilter)

321 removevariables = cache_results(removevariables) 322

323 - def filteraccelerators(self, str1):

324 """filter out accelerators from str1""" 325 return helpers.multifilter(str1, self.accfilters, None)

326 filteraccelerators = cache_results(filteraccelerators) 327

328 - def filteraccelerators_by_list(self, str1, acceptlist=None):

329 """filter out accelerators from str1""" 330 return helpers.multifilter(str1, self.accfilters, acceptlist)

331

332 - def filterwordswithpunctuation(self, str1):

333 """replaces words with punctuation with their unpunctuated 334 equivalents""" 335 return prefilters.filterwordswithpunctuation(str1)

336 filterwordswithpunctuation = cache_results(filterwordswithpunctuation) 337

338 - def filterxml(self, str1):

339 """filter out XML from the string so only text remains""" 340 return tag_re.sub("", str1)

341 filterxml = cache_results(filterxml) 342

343 - def run_test(self, test, unit):

344 """Runs the given test on the given unit. 345 346 Note that this can raise a FilterFailure as part of normal operation""" 347 return test(unit)

348

349 - def run_filters(self, unit):

350 """run all the tests in this suite, return failures as testname, 351 message_or_exception""" 352 self.results_cache = {} 353 failures = {} 354 ignores = self.config.lang.ignoretests[:] 355 functionnames = self.defaultfilters.keys() 356 priorityfunctionnames = self.preconditions.keys() 357 otherfunctionnames = filter(lambda functionname: functionname not in self.preconditions, functionnames) 358 for functionname in priorityfunctionnames + otherfunctionnames: 359 if functionname in ignores: 360 continue 361 filterfunction = getattr(self, functionname, None) 362 # this filterfunction may only be defined on another checker if 363 # using TeeChecker 364 if filterfunction is None: 365 continue 366 filtermessage = filterfunction.__doc__ 367 try: 368 filterresult = self.run_test(filterfunction, unit) 369 except FilterFailure, e: 370 filterresult = False 371 filtermessage = e.args[0] 372 except Exception, e: 373 if self.errorhandler is None: 374 raise ValueError("error in filter %s: %r, %r, %s" % \ 375 (functionname, unit.source, unit.target, e)) 376 else: 377 filterresult = self.errorhandler(functionname, unit.source, 378 unit.target, e) 379 if not filterresult: 380 # we test some preconditions that aren't actually a cause for 381 # failure 382 if functionname in self.defaultfilters: 383 failures[functionname] = filtermessage 384 if functionname in self.preconditions: 385 for ignoredfunctionname in self.preconditions[functionname]: 386 ignores.append(ignoredfunctionname) 387 self.results_cache = {} 388 return failures

389 390

391 -class TranslationChecker(UnitChecker):

392 """A checker that passes source and target strings to the checks, not the 393 whole unit. 394 395 This provides some speedup and simplifies testing.""" 396

397 - def __init__(self, checkerconfig=None, excludefilters=None, 398 limitfilters=None, errorhandler=None):

399 super(TranslationChecker, self).__init__(checkerconfig, excludefilters, 400 limitfilters, errorhandler)

401

402 - def run_test(self, test, unit):

403 """Runs the given test on the given unit. 404 405 Note that this can raise a FilterFailure as part of normal operation.""" 406 if self.hasplural: 407 filtermessages = [] 408 filterresult = True 409 for pluralform in unit.target.strings: 410 try: 411 if not test(self.str1, unicode(pluralform)): 412 filterresult = False 413 except FilterFailure, e: 414 filterresult = False 415 filtermessages.append(unicode(e.args)) 416 if not filterresult and filtermessages: 417 raise FilterFailure(filtermessages) 418 else: 419 return filterresult 420 else: 421 return test(self.str1, self.str2)

422

423 - def run_filters(self, unit):

424 """Do some optimisation by caching some data of the unit for the benefit 425 of run_test().""" 426 self.str1 = data.normalized_unicode(unit.source) or u"" 427 self.str2 = data.normalized_unicode(unit.target) or u"" 428 self.hasplural = unit.hasplural() 429 self.locations = unit.getlocations() 430 return super(TranslationChecker, self).run_filters(unit)

431 432

433 -class TeeChecker:

434 """A Checker that controls multiple checkers.""" 435

436 - def __init__(self, checkerconfig=None, excludefilters=None, 437 limitfilters=None, checkerclasses=None, errorhandler=None, 438 languagecode=None):

439 """construct a TeeChecker from the given checkers""" 440 self.limitfilters = limitfilters 441 if checkerclasses is None: 442 checkerclasses = [StandardChecker] 443 self.checkers = [checkerclass(checkerconfig=checkerconfig, 444 excludefilters=excludefilters, 445 limitfilters=limitfilters, 446 errorhandler=errorhandler) for checkerclass in checkerclasses] 447 if languagecode: 448 for checker in self.checkers: 449 checker.config.updatetargetlanguage(languagecode) 450 # Let's hook up the language specific checker 451 lang_checker = self.checkers[0].config.lang.checker 452 if lang_checker: 453 self.checkers.append(lang_checker) 454 455 self.combinedfilters = self.getfilters(excludefilters, limitfilters) 456 self.config = checkerconfig or self.checkers[0].config

457

458 - def getfilters(self, excludefilters=None, limitfilters=None):

459 """returns dictionary of available filters, including/excluding those in 460 the given lists""" 461 if excludefilters is None: 462 excludefilters = {} 463 filterslist = [checker.getfilters(excludefilters, limitfilters) for checker in self.checkers] 464 self.combinedfilters = {} 465 for filters in filterslist: 466 self.combinedfilters.update(filters) 467 # TODO: move this somewhere more sensible (a checkfilters method?) 468 if limitfilters is not None: 469 for filtername in limitfilters: 470 if not filtername in self.combinedfilters: 471 import sys 472 print >> sys.stderr, "warning: could not find filter %s" % filtername 473 return self.combinedfilters

474

475 - def run_filters(self, unit):

476 """run all the tests in the checker's suites""" 477 failures = {} 478 for checker in self.checkers: 479 failures.update(checker.run_filters(unit)) 480 return failures

481

482 - def setsuggestionstore(self, store):

483 """Sets the filename that a checker should use for evaluating 484 suggestions.""" 485 for checker in self.checkers: 486 checker.setsuggestionstore(store)

487 488

489 -class StandardChecker(TranslationChecker):

490 """The basic test suite for source -> target translations.""" 491

492 - def untranslated(self, str1, str2):

493 """checks whether a string has been translated at all""" 494 str2 = prefilters.removekdecomments(str2) 495 return not (len(str1.strip()) > 0 and len(str2) == 0)

496

497 - def unchanged(self, str1, str2):

498 """checks whether a translation is basically identical to the original 499 string""" 500 str1 = self.filteraccelerators(self.removevariables(str1)).strip() 501 str2 = self.filteraccelerators(self.removevariables(str2)).strip() 502 if len(str1) < 2: 503 return True 504 # If the whole string is upperase, or nothing in the string can go 505 # towards uppercase, let's assume there is nothing translatable 506 # TODO: reconsider 507 if (str1.isupper() or str1.upper() == str1) and str1 == str2: 508 return True 509 if self.config.notranslatewords: 510 words1 = str1.split() 511 if len(words1) == 1 and [word for word in words1 if word in self.config.notranslatewords]: 512 #currently equivalent to: 513 # if len(words1) == 1 and words1[0] in self.config.notranslatewords: 514 #why do we only test for one notranslate word? 515 return True 516 # we could also check for things like str1.isnumeric(), but the test 517 # above (str1.upper() == str1) makes this unnecessary 518 if str1.lower() == str2.lower(): 519 raise FilterFailure(u"please translate") 520 return True

521

522 - def blank(self, str1, str2):

523 """checks whether a translation only contains spaces""" 524 len1 = len(str1.strip()) 525 len2 = len(str2.strip()) 526 return not (len1 > 0 and len(str2) != 0 and len2 == 0)

527

528 - def short(self, str1, str2):

529 """checks whether a translation is much shorter than the original 530 string""" 531 len1 = len(str1.strip()) 532 len2 = len(str2.strip()) 533 return not ((len1 > 0) and (0 < len2 < (len1 * 0.1)) or ((len1 > 1) and (len2 == 1)))

534

535 - def long(self, str1, str2):

536 """checks whether a translation is much longer than the original 537 string""" 538 len1 = len(str1.strip()) 539 len2 = len(str2.strip()) 540 return not ((len1 > 0) and (0 < len1 < (len2 * 0.1)) or ((len1 == 1) and (len2 > 1)))

541

542 - def escapes(self, str1, str2):

543 """checks whether escaping is consistent between the two strings""" 544 if not helpers.countsmatch(str1, str2, (u"\\", u"\\\\")): 545 escapes1 = u", ".join([u"'%s'" % word for word in str1.split() if u"\\" in word]) 546 escapes2 = u", ".join([u"'%s'" % word for word in str2.split() if u"\\" in word]) 547 raise SeriousFilterFailure(u"escapes in original (%s) don't match " 548 "escapes in translation (%s)" % 549 (escapes1, escapes2)) 550 else: 551 return True

552

553 - def newlines(self, str1, str2):

554 """checks whether newlines are consistent between the two strings""" 555 if not helpers.countsmatch(str1, str2, (u"\n", u"\r")): 556 raise FilterFailure(u"line endings in original don't match " 557 "line endings in translation") 558 else: 559 return True

560

561 - def tabs(self, str1, str2):

562 """checks whether tabs are consistent between the two strings""" 563 if not helpers.countmatch(str1, str2, "\t"): 564 raise SeriousFilterFailure(u"tabs in original don't match " 565 "tabs in translation") 566 else: 567 return True

568

569 - def singlequoting(self, str1, str2):

570 """checks whether singlequoting is consistent between the two strings""" 571 str1 = self.filterwordswithpunctuation(self.filteraccelerators(self.filtervariables(str1))) 572 str1 = self.config.lang.punctranslate(str1) 573 str2 = self.filterwordswithpunctuation(self.filteraccelerators(self.filtervariables(str2))) 574 return helpers.countsmatch(str1, str2, (u"'", u"''", u"\\'"))

575

576 - def doublequoting(self, str1, str2):

577 """checks whether doublequoting is consistent between the two strings""" 578 str1 = self.filteraccelerators(self.filtervariables(str1)) 579 str1 = self.filterxml(str1) 580 str1 = self.config.lang.punctranslate(str1) 581 str2 = self.filteraccelerators(self.filtervariables(str2)) 582 str2 = self.filterxml(str2) 583 return helpers.countsmatch(str1, str2, (u'"', u'""', u'\\"', u"«", 584 u"»", u"“", u"”"))

585

586 - def doublespacing(self, str1, str2):

587 """checks for bad double-spaces by comparing to original""" 588 str1 = self.filteraccelerators(str1) 589 str2 = self.filteraccelerators(str2) 590 return helpers.countmatch(str1, str2, u" ")

591

592 - def puncspacing(self, str1, str2):

593 """checks for bad spacing after punctuation""" 594 # Convert all nbsp to space, and just check spaces. Useful intermediate 595 # step to stricter nbsp checking? 596 str1 = self.filteraccelerators(self.filtervariables(str1)) 597 str1 = self.config.lang.punctranslate(str1) 598 str1 = str1.replace(u"\u00a0", u" ") 599 if str1.find(u" ") == -1: 600 return True 601 str2 = self.filteraccelerators(self.filtervariables(str2)) 602 str2 = str2.replace(u"\u00a0", u" ") 603 for puncchar in self.config.punctuation: 604 plaincount1 = str1.count(puncchar) 605 if not plaincount1: 606 continue 607 plaincount2 = str2.count(puncchar) 608 if plaincount1 != plaincount2: 609 continue 610 spacecount1 = str1.count(puncchar + u" ") 611 spacecount2 = str2.count(puncchar + u" ") 612 if spacecount1 != spacecount2: 613 # handle extra spaces that are because of transposed punctuation 614 if abs(spacecount1 - spacecount2) == 1 and str1.endswith(puncchar) != str2.endswith(puncchar): 615 continue 616 return False 617 return True

618

619 - def printf(self, str1, str2):

620 """checks whether printf format strings match""" 621 count1 = count2 = plural = None 622 # self.hasplural only set by run_filters, not always available 623 if 'hasplural' in self.__dict__: 624 plural = self.hasplural 625 for var_num2, match2 in enumerate(printf_pat.finditer(str2)): 626 count2 = var_num2 + 1 627 str2key = match2.group('key') 628 if match2.group('ord'): 629 for var_num1, match1 in enumerate(printf_pat.finditer(str1)): 630 count1 = var_num1 + 1 631 if int(match2.group('ord')) == var_num1 + 1: 632 if match2.group('fullvar') != match1.group('fullvar'): 633 return 0 634 elif str2key: 635 str1key = None 636 for var_num1, match1 in enumerate(printf_pat.finditer(str1)): 637 count1 = var_num1 + 1 638 if match1.group('key') and str2key == match1.group('key'): 639 str1key = match1.group('key') 640 # '%.0s' "placeholder" in plural will match anything 641 if plural and match2.group('fullvar') == '.0s': 642 continue 643 if match1.group('fullvar') != match2.group('fullvar'): 644 return 0 645 if str1key == None: 646 return 0 647 else: 648 for var_num1, match1 in enumerate(printf_pat.finditer(str1)): 649 count1 = var_num1 + 1 650 # '%.0s' "placeholder" in plural will match anything 651 if plural and match2.group('fullvar') == '.0s': 652 continue 653 if (var_num1 == var_num2) and (match1.group('fullvar') != match2.group('fullvar')): 654 return 0 655 656 if count2 is None: 657 if list(printf_pat.finditer(str1)): 658 return 0 659 660 if (count1 or count2) and (count1 != count2): 661 return 0 662 return 1

663

664 - def accelerators(self, str1, str2):

665 """checks whether accelerators are consistent between the two strings""" 666 str1 = self.filtervariables(str1) 667 str2 = self.filtervariables(str2) 668 messages = [] 669 for accelmarker in self.config.accelmarkers: 670 counter1 = decoration.countaccelerators(accelmarker, self.config.sourcelang.validaccel) 671 counter2 = decoration.countaccelerators(accelmarker, self.config.lang.validaccel) 672 count1, countbad1 = counter1(str1) 673 count2, countbad2 = counter2(str2) 674 getaccel = decoration.getaccelerators(accelmarker, self.config.lang.validaccel) 675 accel2, bad2 = getaccel(str2) 676 if count1 == count2: 677 continue 678 if count1 == 1 and count2 == 0: 679 if countbad2 == 1: 680 messages.append(u"accelerator %s appears before an invalid " 681 "accelerator character '%s' (eg. space)" % 682 (accelmarker, bad2[0])) 683 else: 684 messages.append(u"accelerator %s is missing from translation" % 685 accelmarker) 686 elif count1 == 0: 687 messages.append(u"accelerator %s does not occur in original " 688 "and should not be in translation" % accelmarker) 689 elif count1 == 1 and count2 > count1: 690 messages.append(u"accelerator %s is repeated in translation" % 691 accelmarker) 692 else: 693 messages.append(u"accelerator %s occurs %d time(s) in original " 694 "and %d time(s) in translation" % 695 (accelmarker, count1, count2)) 696 if messages: 697 if "accelerators" in self.config.criticaltests: 698 raise SeriousFilterFailure(messages) 699 else: 700 raise FilterFailure(messages) 701 return True

702 703 # def acceleratedvariables(self, str1, str2): 704 # """checks that no variables are accelerated""" 705 # messages = [] 706 # for accelerator in self.config.accelmarkers: 707 # for variablestart, variableend in self.config.varmatches: 708 # error = accelerator + variablestart 709 # if str1.find(error) >= 0: 710 # messages.append(u"original has an accelerated variable") 711 # if str2.find(error) >= 0: 712 # messages.append(u"translation has an accelerated variable") 713 # if messages: 714 # raise FilterFailure(messages) 715 # return True 716

717 - def variables(self, str1, str2):

718 """checks whether variables of various forms are consistent between the 719 two strings""" 720 messages = [] 721 mismatch1, mismatch2 = [], [] 722 varnames1, varnames2 = [], [] 723 for startmarker, endmarker in self.config.varmatches: 724 varchecker = decoration.getvariables(startmarker, endmarker) 725 if startmarker and endmarker: 726 if isinstance(endmarker, int): 727 redecorate = lambda var: startmarker + var 728 else: 729 redecorate = lambda var: startmarker + var + endmarker 730 elif startmarker: 731 redecorate = lambda var: startmarker + var 732 else: 733 redecorate = lambda var: var 734 vars1 = varchecker(str1) 735 vars2 = varchecker(str2) 736 if vars1 != vars2: 737 # we use counts to compare so we can handle multiple variables 738 vars1, vars2 = [var for var in vars1 if vars1.count(var) > vars2.count(var)], \ 739 [var for var in vars2 if vars1.count(var) < vars2.count(var)] 740 # filter variable names we've already seen, so they aren't 741 # matched by more than one filter... 742 vars1, vars2 = [var for var in vars1 if var not in varnames1], [var for var in vars2 if var not in varnames2] 743 varnames1.extend(vars1) 744 varnames2.extend(vars2) 745 vars1 = map(redecorate, vars1) 746 vars2 = map(redecorate, vars2) 747 mismatch1.extend(vars1) 748 mismatch2.extend(vars2) 749 if mismatch1: 750 messages.append(u"do not translate: %s" % u", ".join(mismatch1)) 751 elif mismatch2: 752 messages.append(u"translation contains variables not in original: %s" % u", ".join(mismatch2)) 753 if messages and mismatch1: 754 raise SeriousFilterFailure(messages) 755 elif messages: 756 raise FilterFailure(messages) 757 return True

758

759 - def functions(self, str1, str2):

760 """checks that function names are not translated""" 761 # We can't just use helpers.funcmatch() since it doesn't ignore order 762 return not set(decoration.getfunctions(str1)).symmetric_difference(set(decoration.getfunctions(str2)))

763

764 - def emails(self, str1, str2):

765 """checks that emails are not translated""" 766 return helpers.funcmatch(str1, str2, decoration.getemails)

767

768 - def urls(self, str1, str2):

769 """checks that URLs are not translated""" 770 return helpers.funcmatch(str1, str2, decoration.geturls)

771

772 - def numbers(self, str1, str2):

773 """checks whether numbers of various forms are consistent between the 774 two strings""" 775 return helpers.countsmatch(str1, str2, decoration.getnumbers(str1))

776

777 - def startwhitespace(self, str1, str2):

778 """checks whether whitespace at the beginning of the strings matches""" 779 return helpers.funcmatch(str1, str2, decoration.spacestart)

780

781 - def endwhitespace(self, str1, str2):

782 """checks whether whitespace at the end of the strings matches""" 783 str1 = self.config.lang.punctranslate(str1) 784 return helpers.funcmatch(str1, str2, decoration.spaceend)

785

786 - def startpunc(self, str1, str2):

787 """checks whether punctuation at the beginning of the strings match""" 788 str1 = self.filterxml(self.filterwordswithpunctuation(self.filteraccelerators(self.filtervariables(str1)))) 789 str1 = self.config.lang.punctranslate(str1) 790 str2 = self.filterxml(self.filterwordswithpunctuation(self.filteraccelerators(self.filtervariables(str2)))) 791 return helpers.funcmatch(str1, str2, decoration.puncstart, self.config.punctuation)

792

793 - def endpunc(self, str1, str2):

794 """checks whether punctuation at the end of the strings match""" 795 str1 = self.filtervariables(str1) 796 str1 = self.config.lang.punctranslate(str1) 797 str2 = self.filtervariables(str2) 798 str1 = str1.rstrip() 799 str2 = str2.rstrip() 800 return helpers.funcmatch(str1, str2, decoration.puncend, self.config.endpunctuation + u":")

801

802 - def purepunc(self, str1, str2):

803 """checks that strings that are purely punctuation are not changed""" 804 # this test is a subset of startandend 805 if (decoration.ispurepunctuation(str1)): 806 return str1 == str2 807 else: 808 return not decoration.ispurepunctuation(str2)

809

810 - def brackets(self, str1, str2):

811 """checks that the number of brackets in both strings match""" 812 str1 = self.filtervariables(str1) 813 str2 = self.filtervariables(str2) 814 messages = [] 815 missing = [] 816 extra = [] 817 for bracket in (u"[", u"]", u"{", u"}", u"(", u")"): 818 count1 = str1.count(bracket) 819 count2 = str2.count(bracket) 820 if count2 < count1: 821 missing.append(u"'%s'" % bracket) 822 elif count2 > count1: 823 extra.append(u"'%s'" % bracket) 824 if missing: 825 messages.append(u"translation is missing %s" % u", ".join(missing)) 826 if extra: 827 messages.append(u"translation has extra %s" % u", ".join(extra)) 828 if messages: 829 raise FilterFailure(messages) 830 return True

831

832 - def sentencecount(self, str1, str2):

833 """checks that the number of sentences in both strings match""" 834 str1 = self.filteraccelerators(str1) 835 str2 = self.filteraccelerators(str2) 836 sentences1 = len(self.config.sourcelang.sentences(str1)) 837 sentences2 = len(self.config.lang.sentences(str2)) 838 if not sentences1 == sentences2: 839 raise FilterFailure(u"The number of sentences differ: " 840 "%d versus %d" % (sentences1, sentences2)) 841 return True

842

843 - def options(self, str1, str2):

844 """checks that options are not translated""" 845 str1 = self.filtervariables(str1) 846 for word1 in str1.split(): 847 if word1 != u"--" and word1.startswith(u"--") and word1[-1].isalnum(): 848 parts = word1.split(u"=") 849 if not parts[0] in str2: 850 raise FilterFailure(u"The option %s does not occur or is " 851 "translated in the translation." % parts[0]) 852 if len(parts) > 1 and parts[1] in str2: 853 raise FilterFailure(u"The parameter %(param)s in option %(option)s " 854 "is not translated." % {"param": parts[1], 855 "option": parts[0]}) 856 return True

857

858 - def startcaps(self, str1, str2):

859 """checks that the message starts with the correct capitalisation""" 860 str1 = self.filteraccelerators(str1) 861 str2 = self.filteraccelerators(str2) 862 if len(str1) > 1 and len(str2) > 1: 863 return self.config.sourcelang.capsstart(str1) == self.config.lang.capsstart(str2) 864 if len(str1) == 0 and len(str2) == 0: 865 return True 866 if len(str1) == 0 or len(str2) == 0: 867 return False 868 return True

869

870 - def simplecaps(self, str1, str2):

871 """checks the capitalisation of two strings isn't wildly different""" 872 str1 = self.removevariables(str1) 873 str2 = self.removevariables(str2) 874 # TODO: review this. The 'I' is specific to English, so it probably 875 # serves no purpose to get sourcelang.sentenceend 876 str1 = re.sub(u"[^%s]( I )" % self.config.sourcelang.sentenceend, u" i ", str1) 877 capitals1 = helpers.filtercount(str1, unicode.isupper) 878 capitals2 = helpers.filtercount(str2, unicode.isupper) 879 alpha1 = helpers.filtercount(str1, unicode.isalpha) 880 alpha2 = helpers.filtercount(str2, unicode.isalpha) 881 # Capture the all caps case 882 if capitals1 == alpha1: 883 return capitals2 == alpha2 884 # some heuristic tests to try and see that the style of capitals is 885 # vaguely the same 886 if capitals1 == 0 or capitals1 == 1: 887 return capitals2 == capitals1 888 elif capitals1 < len(str1) / 10: 889 return capitals2 <= len(str2) / 8 890 elif len(str1) < 10: 891 return abs(capitals1 - capitals2) < 3 892 elif capitals1 > len(str1) * 6 / 10: 893 return capitals2 > len(str2) * 6 / 10 894 else: 895 return abs(capitals1 - capitals2) < (len(str1) + len(str2)) / 6

896

897 - def acronyms(self, str1, str2):

898 """checks that acronyms that appear are unchanged""" 899 acronyms = [] 900 allowed = [] 901 for startmatch, endmatch in self.config.varmatches: 902 allowed += decoration.getvariables(startmatch, endmatch)(str1) 903 allowed += self.config.musttranslatewords.keys() 904 str1 = self.filteraccelerators(self.filtervariables(str1)) 905 iter = self.config.lang.word_iter(str1) 906 str2 = self.filteraccelerators(self.filtervariables(str2)) 907 #TODO: strip XML? - should provide better error messsages 908 # see mail/chrome/messanger/smime.properties.po 909 #TODO: consider limiting the word length for recognising acronyms to 910 #something like 5/6 characters 911 for word in iter: 912 if word.isupper() and len(word) > 1 and word not in allowed: 913 if str2.find(word) == -1: 914 acronyms.append(word) 915 if acronyms: 916 raise FilterFailure(u"acronyms should not be translated: %s" % 917 u", ".join(acronyms)) 918 return True

919

920 - def doublewords(self, str1, str2):

921 """checks for repeated words in the translation""" 922 lastword = "" 923 without_newlines = "\n".join(str2.split("\n")) 924 words = self.filteraccelerators(self.removevariables(self.filterxml(without_newlines))).replace(u".", u"").lower().split() 925 for word in words: 926 if word == lastword and word not in self.config.lang.validdoublewords: 927 raise FilterFailure(u"The word '%s' is repeated" % word) 928 lastword = word 929 return True

930

931 - def notranslatewords(self, str1, str2):

932 """checks that words configured as untranslatable appear in the 933 translation too""" 934 if not self.config.notranslatewords: 935 return True 936 str1 = self.filtervariables(str1) 937 str2 = self.filtervariables(str2) 938 #The above is full of strange quotes and things in utf-8 encoding. 939 #single apostrophe perhaps problematic in words like "doesn't" 940 for seperator in self.config.punctuation: 941 str1 = str1.replace(seperator, u" ") 942 str2 = str2.replace(seperator, u" ") 943 words1 = self.filteraccelerators(str1).split() 944 words2 = self.filteraccelerators(str2).split() 945 stopwords = [word for word in words1 if word in self.config.notranslatewords and word not in words2] 946 if stopwords: 947 raise FilterFailure(u"do not translate: %s" % 948 (u", ".join(stopwords))) 949 return True

950

951 - def musttranslatewords(self, str1, str2):

952 """checks that words configured as definitely translatable don't appear 953 in the translation""" 954 if not self.config.musttranslatewords: 955 return True 956 str1 = self.removevariables(str1) 957 str2 = self.removevariables(str2) 958 # The above is full of strange quotes and things in utf-8 encoding. 959 # single apostrophe perhaps problematic in words like "doesn't" 960 for seperator in self.config.punctuation: 961 str1 = str1.replace(seperator, u" ") 962 str2 = str2.replace(seperator, u" ") 963 words1 = self.filteraccelerators(str1).split() 964 words2 = self.filteraccelerators(str2).split() 965 stopwords = [word for word in words1 if word in self.config.musttranslatewords and word in words2] 966 if stopwords: 967 raise FilterFailure(u"please translate: %s" % (u", ".join(stopwords))) 968 return True

969

970 - def validchars(self, str1, str2):

971 """checks that only characters specified as valid appear in the 972 translation""" 973 if not self.config.validcharsmap: 974 return True 975 invalid1 = str1.translate(self.config.validcharsmap) 976 invalid2 = str2.translate(self.config.validcharsmap) 977 invalidchars = [u"'%s' (\\u%04x)" % (invalidchar, ord(invalidchar)) for invalidchar in invalid2 if invalidchar not in invalid1] 978 if invalidchars: 979 raise FilterFailure(u"invalid chars: %s" % (u", ".join(invalidchars))) 980 return True

981

982 - def filepaths(self, str1, str2):

983 """checks that file paths have not been translated""" 984 for word1 in self.filteraccelerators(str1).split(): 985 if word1.startswith(u"/"): 986 if not helpers.countsmatch(str1, str2, (word1,)): 987 return False 988 return True

989

990 - def xmltags(self, str1, str2):

991 """checks that XML/HTML tags have not been translated""" 992 tags1 = tag_re.findall(str1) 993 if len(tags1) > 0: 994 if (len(tags1[0]) == len(str1)) and not u"=" in tags1[0]: 995 return True 996 tags2 = tag_re.findall(str2) 997 properties1 = tagproperties(tags1, self.config.ignoretags) 998 properties2 = tagproperties(tags2, self.config.ignoretags) 999 filtered1 = [] 1000 filtered2 = [] 1001 for property1 in properties1: 1002 filtered1 += [intuplelist(property1, self.config.canchangetags)] 1003 for property2 in properties2: 1004 filtered2 += [intuplelist(property2, self.config.canchangetags)] 1005 1006 # TODO: consider the consequences of different ordering of 1007 # attributes/tags 1008 if filtered1 != filtered2: 1009 return False 1010 else: 1011 # No tags in str1, let's just check that none were added in str2. 1012 # This might be useful for fuzzy strings wrongly unfuzzied. 1013 tags2 = tag_re.findall(str2) 1014 if len(tags2) > 0: 1015 return False 1016 return True

1017

1018 - def kdecomments(self, str1, str2):

1019 """checks to ensure that no KDE style comments appear in the 1020 translation""" 1021 return str2.find(u"\n_:") == -1 and not str2.startswith(u"_:")

1022

1023 - def compendiumconflicts(self, str1, str2):

1024 """checks for Gettext compendium conflicts (#-#-#-#-#)""" 1025 return str2.find(u"#-#-#-#-#") == -1

1026

1027 - def simpleplurals(self, str1, str2):

1028 """checks for English style plural(s) for you to review""" 1029 1030 def numberofpatterns(string, patterns): 1031 number = 0 1032 for pattern in patterns: 1033 number += len(re.findall(pattern, string)) 1034 return number

1035 1036 sourcepatterns = ["$s$"] 1037 targetpatterns = ["$s$"] 1038 sourcecount = numberofpatterns(str1, sourcepatterns) 1039 targetcount = numberofpatterns(str2, targetpatterns) 1040 if self.config.lang.nplurals == 1: 1041 return not targetcount 1042 return sourcecount == targetcount

1043

1044 - def spellcheck(self, str1, str2):

1045 """checks words that don't pass a spell check""" 1046 if not self.config.targetlanguage: 1047 return True 1048 if not spelling.available: 1049 return True 1050 # TODO: filterxml? 1051 str1 = self.filteraccelerators_by_list(self.filtervariables(str1), 1052 self.config.sourcelang.validaccel) 1053 str2 = self.filteraccelerators_by_list(self.filtervariables(str2), 1054 self.config.lang.validaccel) 1055 ignore1 = [] 1056 messages = [] 1057 for word, index, suggestions in spelling.check(str1, lang="en"): 1058 ignore1.append(word) 1059 for word, index, suggestions in spelling.check(str2, lang=self.config.targetlanguage): 1060 if word in self.config.notranslatewords: 1061 continue 1062 if word in ignore1: 1063 continue 1064 # hack to ignore hyphenisation rules 1065 if word in suggestions: 1066 continue 1067 messages.append(u"check spelling of %s (could be %s)" % 1068 (word, u" / ".join(suggestions[:5]))) 1069 if messages: 1070 raise FilterFailure(messages) 1071 return True

1072

1073 - def credits(self, str1, str2):

1074 """checks for messages containing translation credits instead of normal 1075 translations.""" 1076 return not str1 in self.config.credit_sources

1077 1078 # If the precondition filter is run and fails then the other tests listed are ignored 1079 preconditions = { 1080 "untranslated": ("simplecaps", "variables", "startcaps", 1081 "accelerators", "brackets", "endpunc", 1082 "acronyms", "xmltags", "startpunc", 1083 "endwhitespace", "startwhitespace", 1084 "escapes", "doublequoting", "singlequoting", 1085 "filepaths", "purepunc", "doublespacing", 1086 "sentencecount", "numbers", "isfuzzy", 1087 "isreview", "notranslatewords", "musttranslatewords", 1088 "emails", "simpleplurals", "urls", "printf", 1089 "tabs", "newlines", "functions", "options", 1090 "blank", "nplurals", "gconf"), 1091 "blank": ("simplecaps", "variables", "startcaps", 1092 "accelerators", "brackets", "endpunc", 1093 "acronyms", "xmltags", "startpunc", 1094 "endwhitespace", "startwhitespace", 1095 "escapes", "doublequoting", "singlequoting", 1096 "filepaths", "purepunc", "doublespacing", 1097 "sentencecount", "numbers", "isfuzzy", 1098 "isreview", "notranslatewords", "musttranslatewords", 1099 "emails", "simpleplurals", "urls", "printf", 1100 "tabs", "newlines", "functions", "options", 1101 "gconf"), 1102 "credits": ("simplecaps", "variables", "startcaps", 1103 "accelerators", "brackets", "endpunc", 1104 "acronyms", "xmltags", "startpunc", 1105 "escapes", "doublequoting", "singlequoting", 1106 "filepaths", "doublespacing", 1107 "sentencecount", "numbers", 1108 "emails", "simpleplurals", "urls", "printf", 1109 "tabs", "newlines", "functions", "options"), 1110 "purepunc": ("startcaps", "options"), 1111 # This is causing some problems since Python 2.6, as 1112 # startcaps is now seen as an important one to always execute 1113 # and could now be done before it is blocked by a failing 1114 # "untranslated" or "blank" test. This is probably happening 1115 # due to slightly different implementation of the internal 1116 # dict handling since Python 2.6. We should never have relied 1117 # on this ordering anyway. 1118 #"startcaps": ("simplecaps",), 1119 "endwhitespace": ("endpunc",), 1120 "startwhitespace": ("startpunc",), 1121 "unchanged": ("doublewords",), 1122 "compendiumconflicts": ("accelerators", "brackets", "escapes", 1123 "numbers", "startpunc", "long", "variables", 1124 "startcaps", "sentencecount", "simplecaps", 1125 "doublespacing", "endpunc", "xmltags", 1126 "startwhitespace", "endwhitespace", 1127 "singlequoting", "doublequoting", 1128 "filepaths", "purepunc", "doublewords", "printf"), 1129 } 1130 1131 # code to actually run the tests (use unittest?) 1132 1133 openofficeconfig = CheckerConfig( 1134 accelmarkers=["~"], 1135 varmatches=[("&", ";"), ("%", "%"), ("%", None), ("%", 0), ("$(", ")"), 1136 ("$", "$"), ("${", "}"), ("#", "#"), ("#", 1), ("#", 0), 1137 ("($", ")"), ("$[", "]"), ("[", "]"), ("$", None)], 1138 ignoretags=[("alt", "xml-lang", None), ("ahelp", "visibility", "visible"), 1139 ("img", "width", None), ("img", "height", None)], 1140 canchangetags=[("link", "name", None)], 1141 ) 1142 1143

1144 -class OpenOfficeChecker(StandardChecker):

1145

1146 - def __init__(self, **kwargs):

1147 checkerconfig = kwargs.get("checkerconfig", None) 1148 if checkerconfig is None: 1149 checkerconfig = CheckerConfig() 1150 kwargs["checkerconfig"] = checkerconfig 1151 checkerconfig.update(openofficeconfig) 1152 StandardChecker.__init__(self, **kwargs)

1153 1154 mozillaconfig = CheckerConfig( 1155 accelmarkers=["&"], 1156 varmatches=[("&", ";"), ("%", "%"), ("%", 1), ("$", "$"), ("$", None), 1157 ("#", 1), ("${", "}"), ("$(^", ")")], 1158 criticaltests=["accelerators"], 1159 ) 1160 1161

1162 -class MozillaChecker(StandardChecker):

1163

1164 - def __init__(self, **kwargs):

1165 checkerconfig = kwargs.get("checkerconfig", None) 1166 if checkerconfig is None: 1167 checkerconfig = CheckerConfig() 1168 kwargs["checkerconfig"] = checkerconfig 1169 checkerconfig.update(mozillaconfig) 1170 StandardChecker.__init__(self, **kwargs)

1171

1172 - def credits(self, str1, str2):

1173 """checks for messages containing translation credits instead of normal 1174 translations.""" 1175 for location in self.locations: 1176 if location in ['MOZ_LANGPACK_CONTRIBUTORS', 'credit.translation']: 1177 return False 1178 return True

1179 1180 drupalconfig = CheckerConfig( 1181 varmatches=[("%", None), ("@", None), ("!", None)], 1182 ) 1183 1184

1185 -class DrupalChecker(StandardChecker):

1186

1187 - def __init__(self, **kwargs):

1188 checkerconfig = kwargs.get("checkerconfig", None) 1189 if checkerconfig is None: 1190 checkerconfig = CheckerConfig() 1191 kwargs["checkerconfig"] = checkerconfig 1192 checkerconfig.update(drupalconfig) 1193 StandardChecker.__init__(self, **kwargs)

1194 1195 gnomeconfig = CheckerConfig( 1196 accelmarkers=["_"], 1197 varmatches=[("%", 1), ("$(", ")")], 1198 credit_sources=[u"translator-credits"], 1199 ) 1200 1201

1202 -class GnomeChecker(StandardChecker):

1203

1204 - def __init__(self, **kwargs):

1205 checkerconfig = kwargs.get("checkerconfig", None) 1206 if checkerconfig is None: 1207 checkerconfig = CheckerConfig() 1208 kwargs["checkerconfig"] = checkerconfig 1209 checkerconfig.update(gnomeconfig) 1210 StandardChecker.__init__(self, **kwargs)

1211

1212 - def gconf(self, str1, str2):

1213 """Checks if we have any gconf config settings translated.""" 1214 for location in self.locations: 1215 if location.find('schemas.in') != -1: 1216 gconf_attributes = gconf_attribute_re.findall(str1) 1217 #stopwords = [word for word in words1 if word in self.config.notranslatewords and word not in words2] 1218 stopwords = [word for word in gconf_attributes if word[1:-1] not in str2] 1219 if stopwords: 1220 raise FilterFailure(u"do not translate gconf attribute: %s" % 1221 (u", ".join(stopwords))) 1222 return True

1223 1224 kdeconfig = CheckerConfig( 1225 accelmarkers=["&"], 1226 varmatches=[("%", 1)], 1227 credit_sources=[u"Your names", u"Your emails", u"ROLES_OF_TRANSLATORS"], 1228 ) 1229 1230

1231 -class KdeChecker(StandardChecker):

1232

1233 - def __init__(self, **kwargs):

1234 # TODO allow setup of KDE plural and translator comments so that they do 1235 # not create false postives 1236 checkerconfig = kwargs.get("checkerconfig", None) 1237 if checkerconfig is None: 1238 checkerconfig = CheckerConfig() 1239 kwargs["checkerconfig"] = checkerconfig 1240 checkerconfig.update(kdeconfig) 1241 StandardChecker.__init__(self, **kwargs)

1242 1243 cclicenseconfig = CheckerConfig(varmatches=[("@", "@")]) 1244 1245

1246 -class CCLicenseChecker(StandardChecker):

1247

1248 - def __init__(self, **kwargs):

1249 checkerconfig = kwargs.get("checkerconfig", None) 1250 if checkerconfig is None: 1251 checkerconfig = CheckerConfig() 1252 kwargs["checkerconfig"] = checkerconfig 1253 checkerconfig.update(cclicenseconfig) 1254 StandardChecker.__init__(self, **kwargs)

1255 1256 projectcheckers = { 1257 "openoffice": OpenOfficeChecker, 1258 "mozilla": MozillaChecker, 1259 "kde": KdeChecker, 1260 "wx": KdeChecker, 1261 "gnome": GnomeChecker, 1262 "creativecommons": CCLicenseChecker, 1263 "drupal": DrupalChecker, 1264 } 1265 1266

1267 -class StandardUnitChecker(UnitChecker):

1268 """The standard checks for common checks on translation units.""" 1269

1270 - def isfuzzy(self, unit):

1271 """Check if the unit has been marked fuzzy.""" 1272 return not unit.isfuzzy()

1273

1274 - def isreview(self, unit):

1275 """Check if the unit has been marked review.""" 1276 return not unit.isreview()

1277

1278 - def nplurals(self, unit):

1279 """Checks for the correct number of noun forms for plural 1280 translations.""" 1281 if unit.hasplural(): 1282 # if we don't have a valid nplurals value, don't run the test 1283 nplurals = self.config.lang.nplurals 1284 if nplurals > 0: 1285 return len(unit.target.strings) == nplurals 1286 return True

1287

1288 - def hassuggestion(self, unit):

1289 """Checks if there is at least one suggested translation for this 1290 unit.""" 1291 self.suggestion_store = getattr(self, 'suggestion_store', None) 1292 suggestions = [] 1293 if self.suggestion_store: 1294 suggestions = self.suggestion_store.findunits(unit.source) 1295 elif xliff and isinstance(unit, xliff.xliffunit): 1296 # TODO: we probably want to filter them somehow 1297 suggestions = unit.getalttrans() 1298 return not bool(suggestions)

1299 1300

1301 -def runtests(str1, str2, ignorelist=()):

1302 """verifies that the tests pass for a pair of strings""" 1303 from translate.storage import base 1304 str1 = data.normalized_unicode(str1) 1305 str2 = data.normalized_unicode(str2) 1306 unit = base.TranslationUnit(str1) 1307 unit.target = str2 1308 checker = StandardChecker(excludefilters=ignorelist) 1309 failures = checker.run_filters(unit) 1310 for test in failures: 1311 print "failure: %s: %s\n %r\n %r" % \ 1312 (test, failures[test], str1, str2) 1313 return failures

1314 1315

1316 -def batchruntests(pairs):

1317 """runs test on a batch of string pairs""" 1318 passed, numpairs = 0, len(pairs) 1319 for str1, str2 in pairs: 1320 if runtests(str1, str2): 1321 passed += 1 1322 print 1323 print "total: %d/%d pairs passed" % (passed, numpairs)

1324 1325 1326 if __name__ == '__main__': 1327 testset = [(r"simple", r"somple"), 1328 (r"\this equals \that", r"does \this equal \that?"), 1329 (r"this \'equals\' that", r"this 'equals' that"), 1330 (r" start and end! they must match.", 1331 r"start and end! they must match."), 1332 (r"check for matching %variables marked like %this", 1333 r"%this %variable is marked"), 1334 (r"check for mismatching %variables marked like %this", 1335 r"%that %variable is marked"), 1336 (r"check for mismatching %variables% too", 1337 r"how many %variable% are marked"), 1338 (r"%% %%", r"%%"), 1339 (r"Row: %1, Column: %2", r"Mothalo: %1, Kholomo: %2"), 1340 (r"simple lowercase", r"it is all lowercase"), 1341 (r"simple lowercase", r"It Is All Lowercase"), 1342 (r"Simple First Letter Capitals", r"First Letters"), 1343 (r"SIMPLE CAPITALS", r"First Letters"), 1344 (r"SIMPLE CAPITALS", r"ALL CAPITALS"), 1345 (r"forgot to translate", r" "), 1346 ] 1347 batchruntests(testset) 1348

Source Code for Module translate.filters.checks