translate.filters.checks

1 #!/usr/bin/env python 2 # -*- coding: utf-8 -*- 3 # 4 # Copyright 2004-2008 Zuza Software Foundation 5 # 6 # This file is part of translate. 7 # 8 # translate is free software; you can redistribute it and/or modify 9 # it under the terms of the GNU General Public License as published by 10 # the Free Software Foundation; either version 2 of the License, or 11 # (at your option) any later version. 12 # 13 # translate is distributed in the hope that it will be useful, 14 # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 # GNU General Public License for more details. 17 # 18 # You should have received a copy of the GNU General Public License 19 # along with translate; if not, write to the Free Software 20 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 21 22 """This is a set of validation checks that can be performed on translation 23 units. 24 25 Derivatives of UnitChecker (like StandardUnitChecker) check translation units, 26 and derivatives of TranslationChecker (like StandardChecker) check 27 (source, target) translation pairs. 28 29 When adding a new test here, please document and explain the behaviour on the 30 U{wiki <http://translate.sourceforge.net/wiki/toolkit/pofilter_tests>}. 31 """ 32 33 from translate.filters import helpers 34 from translate.filters import decoration 35 from translate.filters import prefilters 36 from translate.filters import spelling 37 from translate.lang import factory 38 from translate.lang import data 39 # The import of xliff could fail if the user doesn't have lxml installed. For 40 # now we try to continue gracefully to help users who aren't interested in 41 # support for XLIFF or other XML formats. 42 try: 43 from translate.storage import xliff 44 except ImportError, e: 45 xliff = None 46 # The import of xliff fail silently in the absence of lxml if another module 47 # already tried to import it unsuccessfully, so let's make 100% sure: 48 if not hasattr(xliff, "xliffunit"): 49 xliff = None 50 import re 51 52 # These are some regular expressions that are compiled for use in some tests 53 54 # printf syntax based on http://en.wikipedia.org/wiki/Printf which doens't cover everything we leave \w instead of specifying the exact letters as 55 # this should capture printf types defined in other platforms. 56 # extended to support Python named format specifiers 57 printf_pat = re.compile('%((?:(?P<ord>\d+)\$|$(?P<key>\w+)$)?(?P<fullvar>[+#-]*(?:\d+)?(?:\.\d+)?(hh\|h\|l\|ll)?(?P<type>[\w%])))') 58 59 # The name of the XML tag 60 tagname_re = re.compile("<[\s]*([\w\/]*)") 61 62 # We allow escaped quotes, probably for old escaping style of OOo helpcontent 63 #TODO: remove escaped strings once usage is audited 64 property_re = re.compile(" (\w*)=((\\\\?\".*?\\\\?\")|(\\\\?'.*?\\\\?'))") 65 66 # The whole tag 67 tag_re = re.compile("<[^>]+>") 68 69 gconf_attribute_re = re.compile('"[a-z_]+?"') 70

71 -def tagname(string):

72 """Returns the name of the XML/HTML tag in string""" 73 return tagname_re.match(string).groups(1)[0]

74

75 -def intuplelist(pair, list):

76 """Tests to see if pair == (a,b,c) is in list, but handles None entries in 77 list as wildcards (only allowed in positions "a" and "c"). We take a shortcut 78 by only considering "c" if "b" has already matched.""" 79 a, b, c = pair 80 if (b, c) == (None, None): 81 #This is a tagname 82 return pair 83 for pattern in list: 84 x, y, z = pattern 85 if (x, y) in [(a, b), (None, b)]: 86 if z in [None, c]: 87 return pattern 88 return pair

89

90 -def tagproperties(strings, ignore):

91 """Returns all the properties in the XML/HTML tag string as 92 (tagname, propertyname, propertyvalue), but ignore those combinations 93 specified in ignore.""" 94 properties = [] 95 for string in strings: 96 tag = tagname(string) 97 properties += [(tag, None, None)] 98 #Now we isolate the attribute pairs. 99 pairs = property_re.findall(string) 100 for property, value, a, b in pairs: 101 #Strip the quotes: 102 value = value[1:-1] 103 104 canignore = False 105 if (tag, property, value) in ignore or \ 106 intuplelist((tag,property,value), ignore) != (tag,property,value): 107 canignore = True 108 break 109 if not canignore: 110 properties += [(tag, property, value)] 111 return properties

112 113

114 -class FilterFailure(Exception):

115 """This exception signals that a Filter didn't pass, and gives an explanation 116 or a comment"""

117 - def __init__(self, messages):

118 if not isinstance(messages, list): 119 messages = [messages] 120 assert isinstance(messages[0], unicode) # Assumption: all of same type 121 joined = u", ".join(messages) 122 Exception.__init__(self, joined) 123 # Python 2.3 doesn't have .args 124 if not hasattr(self, "args"): 125 self.args = joined

126

127 -class SeriousFilterFailure(FilterFailure):

128 """This exception signals that a Filter didn't pass, and the bad translation 129 might break an application (so the string will be marked fuzzy)""" 130 pass

131 132 #(tag, attribute, value) specifies a certain attribute which can be changed/ 133 #ignored if it exists inside tag. In the case where there is a third element 134 #in the tuple, it indicates a property value that can be ignored if present 135 #(like defaults, for example) 136 #If a certain item is None, it indicates that it is relevant for all values of 137 #the property/tag that is specified as None. A non-None value of "value" 138 #indicates that the value of the attribute must be taken into account. 139 common_ignoretags = [(None, "xml-lang", None)] 140 common_canchangetags = [("img", "alt", None), (None, "title", None)] 141 # Actually the title tag is allowed on many tags in HTML (but probably not all) 142

143 -class CheckerConfig(object):

144 """object representing the configuration of a checker"""

145 - def __init__(self, targetlanguage=None, accelmarkers=None, varmatches=None, 146 notranslatewords=None, musttranslatewords=None, validchars=None, 147 punctuation=None, endpunctuation=None, ignoretags=None, 148 canchangetags=None, criticaltests=None, credit_sources=None):

149 # Init lists 150 self.accelmarkers = self._init_list(accelmarkers) 151 self.varmatches = self._init_list(varmatches) 152 self.criticaltests = self._init_list(criticaltests) 153 self.credit_sources = self._init_list(credit_sources) 154 # Lang data 155 self.targetlanguage = targetlanguage 156 self.updatetargetlanguage(targetlanguage) 157 self.sourcelang = factory.getlanguage('en') 158 # Inits with default values 159 self.punctuation = self._init_default(data.normalized_unicode(punctuation), self.lang.punctuation) 160 self.endpunctuation = self._init_default(data.normalized_unicode(endpunctuation), self.lang.sentenceend) 161 self.ignoretags = self._init_default(ignoretags, common_ignoretags) 162 self.canchangetags = self._init_default(canchangetags, common_canchangetags) 163 # Other data 164 # TODO: allow user configuration of untranslatable words 165 self.notranslatewords = dict.fromkeys([data.normalized_unicode(key) for key in self._init_list(notranslatewords)]) 166 self.musttranslatewords = dict.fromkeys([data.normalized_unicode(key) for key in self._init_list(musttranslatewords)]) 167 validchars = data.normalized_unicode(validchars) 168 self.validcharsmap = {} 169 self.updatevalidchars(validchars)

170

171 - def _init_list(self, list):

172 """initialise configuration paramaters that are lists 173 174 @type list: List 175 @param list: None (we'll initialise a blank list) or a list paramater 176 @rtype: List 177 """ 178 if list is None: 179 list = [] 180 return list

181

182 - def _init_default(self, param, default):

183 """initialise parameters that can have default options 184 185 @param param: the user supplied paramater value 186 @param default: default values when param is not specified 187 @return: the paramater as specified by the user of the default settings 188 """ 189 if param is None: 190 return default 191 return param

192

193 - def update(self, otherconfig):

194 """combines the info in otherconfig into this config object""" 195 self.targetlanguage = otherconfig.targetlanguage or self.targetlanguage 196 self.updatetargetlanguage(self.targetlanguage) 197 self.accelmarkers.extend([c for c in otherconfig.accelmarkers if not c in self.accelmarkers]) 198 self.varmatches.extend(otherconfig.varmatches) 199 self.notranslatewords.update(otherconfig.notranslatewords) 200 self.musttranslatewords.update(otherconfig.musttranslatewords) 201 self.validcharsmap.update(otherconfig.validcharsmap) 202 self.punctuation += otherconfig.punctuation 203 self.endpunctuation += otherconfig.endpunctuation 204 #TODO: consider also updating in the following cases: 205 self.ignoretags = otherconfig.ignoretags 206 self.canchangetags = otherconfig.canchangetags 207 self.criticaltests.extend(otherconfig.criticaltests) 208 self.credit_sources = otherconfig.credit_sources

209

210 - def updatevalidchars(self, validchars):

211 """updates the map that eliminates valid characters""" 212 if validchars is None: 213 return True 214 validcharsmap = dict([(ord(validchar), None) for validchar in data.normalized_unicode(validchars)]) 215 self.validcharsmap.update(validcharsmap)

216

217 - def updatetargetlanguage(self, langcode):

218 """Updates the target language in the config to the given target language""" 219 self.lang = factory.getlanguage(langcode)

220

221 -def cache_results(f):

222 def cached_f(self, param1): 223 key = (f.__name__, param1) 224 res_cache = self.results_cache 225 if key in res_cache: 226 return res_cache[key] 227 else: 228 value = f(self, param1) 229 res_cache[key] = value 230 return value

231 return cached_f 232

233 -class UnitChecker(object):

234 """Parent Checker class which does the checking based on functions available 235 in derived classes.""" 236 preconditions = {} 237

238 - def __init__(self, checkerconfig=None, excludefilters=None, limitfilters=None, errorhandler=None):

239 self.errorhandler = errorhandler 240 if checkerconfig is None: 241 self.setconfig(CheckerConfig()) 242 else: 243 self.setconfig(checkerconfig) 244 # exclude functions defined in UnitChecker from being treated as tests... 245 self.helperfunctions = {} 246 for functionname in dir(UnitChecker): 247 function = getattr(self, functionname) 248 if callable(function): 249 self.helperfunctions[functionname] = function 250 self.defaultfilters = self.getfilters(excludefilters, limitfilters) 251 252 self.results_cache = {}

253

254 - def getfilters(self, excludefilters=None, limitfilters=None):

255 """returns dictionary of available filters, including/excluding those in 256 the given lists""" 257 filters = {} 258 if limitfilters is None: 259 # use everything available unless instructed 260 limitfilters = dir(self) 261 if excludefilters is None: 262 excludefilters = {} 263 for functionname in limitfilters: 264 if functionname in excludefilters: continue 265 if functionname in self.helperfunctions: continue 266 if functionname == "errorhandler": continue 267 filterfunction = getattr(self, functionname, None) 268 if not callable(filterfunction): continue 269 filters[functionname] = filterfunction 270 return filters

271

272 - def setconfig(self, config):

273 """sets the accelerator list""" 274 self.config = config 275 self.accfilters = [prefilters.filteraccelerators(accelmarker) for accelmarker in self.config.accelmarkers] 276 self.varfilters = [prefilters.filtervariables(startmatch, endmatch, prefilters.varname) 277 for startmatch, endmatch in self.config.varmatches] 278 self.removevarfilter = [prefilters.filtervariables(startmatch, endmatch, prefilters.varnone) 279 for startmatch, endmatch in self.config.varmatches]

280

281 - def setsuggestionstore(self, store):

282 """Sets the filename that a checker should use for evaluating suggestions.""" 283 self.suggestion_store = store 284 if self.suggestion_store: 285 self.suggestion_store.require_index()

286

287 - def filtervariables(self, str1):

288 """filter out variables from str1""" 289 return helpers.multifilter(str1, self.varfilters)

290 filtervariables = cache_results(filtervariables) 291

292 - def removevariables(self, str1):

293 """remove variables from str1""" 294 return helpers.multifilter(str1, self.removevarfilter)

295 removevariables = cache_results(removevariables) 296

297 - def filteraccelerators(self, str1):

298 """filter out accelerators from str1""" 299 return helpers.multifilter(str1, self.accfilters, None)

300 filteraccelerators = cache_results(filteraccelerators) 301

302 - def filteraccelerators_by_list(self, str1, acceptlist=None):

303 """filter out accelerators from str1""" 304 return helpers.multifilter(str1, self.accfilters, acceptlist)

305

306 - def filterwordswithpunctuation(self, str1):

307 """replaces words with punctuation with their unpunctuated equivalents""" 308 return prefilters.filterwordswithpunctuation(str1)

309 filterwordswithpunctuation = cache_results(filterwordswithpunctuation) 310

311 - def filterxml(self, str1):

312 """filter out XML from the string so only text remains""" 313 return tag_re.sub("", str1)

314 filterxml = cache_results(filterxml) 315

316 - def run_test(self, test, unit):

317 """Runs the given test on the given unit. 318 319 Note that this can raise a FilterFailure as part of normal operation""" 320 return test(unit)

321

322 - def run_filters(self, unit):

323 """run all the tests in this suite, return failures as testname, message_or_exception""" 324 self.results_cache = {} 325 failures = {} 326 ignores = self.config.lang.ignoretests[:] 327 functionnames = self.defaultfilters.keys() 328 priorityfunctionnames = self.preconditions.keys() 329 otherfunctionnames = filter(lambda functionname: functionname not in self.preconditions, functionnames) 330 for functionname in priorityfunctionnames + otherfunctionnames: 331 if functionname in ignores: 332 continue 333 filterfunction = getattr(self, functionname, None) 334 # this filterfunction may only be defined on another checker if using TeeChecker 335 if filterfunction is None: 336 continue 337 filtermessage = filterfunction.__doc__ 338 try: 339 filterresult = self.run_test(filterfunction, unit) 340 except FilterFailure, e: 341 filterresult = False 342 filtermessage = e.args[0] 343 except Exception, e: 344 if self.errorhandler is None: 345 raise ValueError("error in filter %s: %r, %r, %s" % \ 346 (functionname, unit.source, unit.target, e)) 347 else: 348 filterresult = self.errorhandler(functionname, unit.source, unit.target, e) 349 if not filterresult: 350 # we test some preconditions that aren't actually a cause for failure 351 if functionname in self.defaultfilters: 352 failures[functionname] = filtermessage 353 if functionname in self.preconditions: 354 for ignoredfunctionname in self.preconditions[functionname]: 355 ignores.append(ignoredfunctionname) 356 self.results_cache = {} 357 return failures

358

359 -class TranslationChecker(UnitChecker):

360 """A checker that passes source and target strings to the checks, not the 361 whole unit. 362 363 This provides some speedup and simplifies testing."""

364 - def __init__(self, checkerconfig=None, excludefilters=None, limitfilters=None, errorhandler=None):

365 super(TranslationChecker, self).__init__(checkerconfig, excludefilters, limitfilters, errorhandler)

366

367 - def run_test(self, test, unit):

368 """Runs the given test on the given unit. 369 370 Note that this can raise a FilterFailure as part of normal operation.""" 371 if self.hasplural: 372 filtermessages = [] 373 filterresult = True 374 for pluralform in unit.target.strings: 375 try: 376 if not test(self.str1, unicode(pluralform)): 377 filterresult = False 378 except FilterFailure, e: 379 filterresult = False 380 filtermessages.append( unicode(e.args) ) 381 if not filterresult and filtermessages: 382 raise FilterFailure(filtermessages) 383 else: 384 return filterresult 385 else: 386 return test(self.str1, self.str2)

387

388 - def run_filters(self, unit):

389 """Do some optimisation by caching some data of the unit for the benefit 390 of run_test().""" 391 self.str1 = data.normalized_unicode(unit.source) or u"" 392 self.str2 = data.normalized_unicode(unit.target) or u"" 393 self.hasplural = unit.hasplural() 394 self.locations = unit.getlocations() 395 return super(TranslationChecker, self).run_filters(unit)

396

397 -class TeeChecker:

398 """A Checker that controls multiple checkers."""

399 - def __init__(self, checkerconfig=None, excludefilters=None, limitfilters=None, 400 checkerclasses=None, errorhandler=None, languagecode=None):

401 """construct a TeeChecker from the given checkers""" 402 self.limitfilters = limitfilters 403 if checkerclasses is None: 404 checkerclasses = [StandardChecker] 405 self.checkers = [checkerclass(checkerconfig=checkerconfig, excludefilters=excludefilters, limitfilters=limitfilters, errorhandler=errorhandler) for checkerclass in checkerclasses] 406 if languagecode: 407 for checker in self.checkers: 408 checker.config.updatetargetlanguage(languagecode) 409 # Let's hook up the language specific checker 410 lang_checker = self.checkers[0].config.lang.checker 411 if lang_checker: 412 self.checkers.append(lang_checker) 413 414 self.combinedfilters = self.getfilters(excludefilters, limitfilters) 415 self.config = checkerconfig or self.checkers[0].config

416

417 - def getfilters(self, excludefilters=None, limitfilters=None):

418 """returns dictionary of available filters, including/excluding those in 419 the given lists""" 420 if excludefilters is None: 421 excludefilters = {} 422 filterslist = [checker.getfilters(excludefilters, limitfilters) for checker in self.checkers] 423 self.combinedfilters = {} 424 for filters in filterslist: 425 self.combinedfilters.update(filters) 426 # TODO: move this somewhere more sensible (a checkfilters method?) 427 if limitfilters is not None: 428 for filtername in limitfilters: 429 if not filtername in self.combinedfilters: 430 import sys 431 print >> sys.stderr, "warning: could not find filter %s" % filtername 432 return self.combinedfilters

433

434 - def run_filters(self, unit):

435 """run all the tests in the checker's suites""" 436 failures = {} 437 for checker in self.checkers: 438 failures.update(checker.run_filters(unit)) 439 return failures

440

441 - def setsuggestionstore(self, store):

442 """Sets the filename that a checker should use for evaluating suggestions.""" 443 for checker in self.checkers: 444 checker.setsuggestionstore(store)

445 446

447 -class StandardChecker(TranslationChecker):

448 """The basic test suite for source -> target translations."""

449 - def untranslated(self, str1, str2):

450 """checks whether a string has been translated at all""" 451 str2 = prefilters.removekdecomments(str2) 452 return not (len(str1.strip()) > 0 and len(str2) == 0)

453

454 - def unchanged(self, str1, str2):

455 """checks whether a translation is basically identical to the original string""" 456 str1 = self.filteraccelerators(self.removevariables(str1)).strip() 457 str2 = self.filteraccelerators(self.removevariables(str2)).strip() 458 if len(str1) < 2: 459 return True 460 # If the whole string is upperase, or nothing in the string can go 461 # towards uppercase, let's assume there is nothing translatable 462 # TODO: reconsider 463 if (str1.isupper() or str1.upper() == str1) and str1 == str2: 464 return True 465 if self.config.notranslatewords: 466 words1 = str1.split() 467 if len(words1) == 1 and [word for word in words1 if word in self.config.notranslatewords]: 468 #currently equivalent to: 469 # if len(words1) == 1 and words1[0] in self.config.notranslatewords: 470 #why do we only test for one notranslate word? 471 return True 472 # we could also check for things like str1.isnumeric(), but the test 473 # above (str1.upper() == str1) makes this unnecessary 474 if str1.lower() == str2.lower(): 475 raise FilterFailure(u"please translate") 476 return True

477

478 - def blank(self, str1, str2):

479 """checks whether a translation only contains spaces""" 480 len1 = len(str1.strip()) 481 len2 = len(str2.strip()) 482 return not (len1 > 0 and len(str2) != 0 and len2 == 0)

483

484 - def short(self, str1, str2):

485 """checks whether a translation is much shorter than the original string""" 486 len1 = len(str1.strip()) 487 len2 = len(str2.strip()) 488 return not ((len1 > 0) and (0 < len2 < (len1 * 0.1)) or ((len1 > 1) and (len2 == 1)))

489

490 - def long(self, str1, str2):

491 """checks whether a translation is much longer than the original string""" 492 len1 = len(str1.strip()) 493 len2 = len(str2.strip()) 494 return not ((len1 > 0) and (0 < len1 < (len2 * 0.1)) or ((len1 == 1) and (len2 > 1)))

495

496 - def escapes(self, str1, str2):

497 """checks whether escaping is consistent between the two strings""" 498 if not helpers.countsmatch(str1, str2, (u"\\", u"\\\\")): 499 escapes1 = u", ".join([u"'%s'" % word for word in str1.split() if u"\\" in word]) 500 escapes2 = u", ".join([u"'%s'" % word for word in str2.split() if u"\\" in word]) 501 raise SeriousFilterFailure(u"escapes in original (%s) don't match escapes in translation (%s)" % (escapes1, escapes2)) 502 else: 503 return True

504

505 - def newlines(self, str1, str2):

506 """checks whether newlines are consistent between the two strings""" 507 if not helpers.countsmatch(str1, str2, (u"\n", u"\r")): 508 raise FilterFailure(u"line endings in original don't match line endings in translation") 509 else: 510 return True

511

512 - def tabs(self, str1, str2):

513 """checks whether tabs are consistent between the two strings""" 514 if not helpers.countmatch(str1, str2, "\t"): 515 raise SeriousFilterFailure(u"tabs in original don't match tabs in translation") 516 else: 517 return True

518

519 - def singlequoting(self, str1, str2):

520 """checks whether singlequoting is consistent between the two strings""" 521 str1 = self.filterwordswithpunctuation(self.filteraccelerators(self.filtervariables(str1))) 522 str1 = self.config.lang.punctranslate(str1) 523 str2 = self.filterwordswithpunctuation(self.filteraccelerators(self.filtervariables(str2))) 524 return helpers.countsmatch(str1, str2, (u"'", u"''", u"\\'"))

525

526 - def doublequoting(self, str1, str2):

527 """checks whether doublequoting is consistent between the two strings""" 528 str1 = self.filteraccelerators(self.filtervariables(str1)) 529 str1 = self.filterxml(str1) 530 str1 = self.config.lang.punctranslate(str1) 531 str2 = self.filteraccelerators(self.filtervariables(str2)) 532 str2 = self.filterxml(str2) 533 return helpers.countsmatch(str1, str2, (u'"', u'""', u'\\"', u"«", u"»", u"“", u"”"))

534

535 - def doublespacing(self, str1, str2):

536 """checks for bad double-spaces by comparing to original""" 537 str1 = self.filteraccelerators(str1) 538 str2 = self.filteraccelerators(str2) 539 return helpers.countmatch(str1, str2, u" ")

540

541 - def puncspacing(self, str1, str2):

542 """checks for bad spacing after punctuation""" 543 # Convert all nbsp to space, and just check spaces. Useful intermediate step to stricter nbsp checking? 544 str1 = self.filteraccelerators(self.filtervariables(str1)) 545 str1 = self.config.lang.punctranslate(str1) 546 str1 = str1.replace(u"\u00a0", u" ") 547 if str1.find(u" ") == -1: 548 return True 549 str2 = self.filteraccelerators(self.filtervariables(str2)) 550 str2 = str2.replace(u"\u00a0", u" ") 551 for puncchar in self.config.punctuation: 552 plaincount1 = str1.count(puncchar) 553 plaincount2 = str2.count(puncchar) 554 if not plaincount1 or plaincount1 != plaincount2: 555 continue 556 spacecount1 = str1.count(puncchar + u" ") 557 spacecount2 = str2.count(puncchar + u" ") 558 if spacecount1 != spacecount2: 559 # handle extra spaces that are because of transposed punctuation 560 if str1.endswith(puncchar) != str2.endswith(puncchar) and abs(spacecount1-spacecount2) == 1: 561 continue 562 return False 563 return True

564

565 - def printf(self, str1, str2):

566 """checks whether printf format strings match""" 567 count1 = count2 = plural = None 568 # self.hasplural only set by run_filters, not always available 569 if 'hasplural' in self.__dict__: 570 plural = self.hasplural 571 for var_num2, match2 in enumerate(printf_pat.finditer(str2)): 572 count2 = var_num2 + 1 573 str2key = match2.group('key') 574 if match2.group('ord'): 575 for var_num1, match1 in enumerate(printf_pat.finditer(str1)): 576 count1 = var_num1 + 1 577 if int(match2.group('ord')) == var_num1 + 1: 578 if match2.group('fullvar') != match1.group('fullvar'): 579 return 0 580 elif str2key: 581 str1key = None 582 for var_num1, match1 in enumerate(printf_pat.finditer(str1)): 583 count1 = var_num1 + 1 584 if match1.group('key') and str2key == match1.group('key'): 585 str1key = match1.group('key') 586 # '%.0s' "placeholder" in plural will match anything 587 if plural and match2.group('fullvar') == '.0s': 588 continue 589 if match1.group('fullvar') != match2.group('fullvar'): 590 return 0 591 if str1key == None: 592 return 0 593 else: 594 for var_num1, match1 in enumerate(printf_pat.finditer(str1)): 595 count1 = var_num1 + 1 596 # '%.0s' "placeholder" in plural will match anything 597 if plural and match2.group('fullvar') == '.0s': 598 continue 599 if (var_num1 == var_num2) and (match1.group('fullvar') != match2.group('fullvar')): 600 return 0 601 602 if count2 is None: 603 if list(printf_pat.finditer(str1)): 604 return 0 605 606 if (count1 or count2) and (count1 != count2): 607 return 0 608 return 1

609

610 - def accelerators(self, str1, str2):

611 """checks whether accelerators are consistent between the two strings""" 612 str1 = self.filtervariables(str1) 613 str2 = self.filtervariables(str2) 614 messages = [] 615 for accelmarker in self.config.accelmarkers: 616 counter1 = decoration.countaccelerators(accelmarker, self.config.sourcelang.validaccel) 617 counter2 = decoration.countaccelerators(accelmarker, self.config.lang.validaccel) 618 count1, countbad1 = counter1(str1) 619 count2, countbad2 = counter2(str2) 620 getaccel = decoration.getaccelerators(accelmarker, self.config.lang.validaccel) 621 accel2, bad2 = getaccel(str2) 622 if count1 == count2: 623 continue 624 if count1 == 1 and count2 == 0: 625 if countbad2 == 1: 626 messages.append(u"accelerator %s appears before an invalid accelerator character '%s' (eg. space)" % (accelmarker, bad2[0])) 627 else: 628 messages.append(u"accelerator %s is missing from translation" % accelmarker) 629 elif count1 == 0: 630 messages.append(u"accelerator %s does not occur in original and should not be in translation" % accelmarker) 631 elif count1 == 1 and count2 > count1: 632 messages.append(u"accelerator %s is repeated in translation" % accelmarker) 633 else: 634 messages.append(u"accelerator %s occurs %d time(s) in original and %d time(s) in translation" % (accelmarker, count1, count2)) 635 if messages: 636 if "accelerators" in self.config.criticaltests: 637 raise SeriousFilterFailure(messages) 638 else: 639 raise FilterFailure(messages) 640 return True

641 642 # def acceleratedvariables(self, str1, str2): 643 # """checks that no variables are accelerated""" 644 # messages = [] 645 # for accelerator in self.config.accelmarkers: 646 # for variablestart, variableend in self.config.varmatches: 647 # error = accelerator + variablestart 648 # if str1.find(error) >= 0: 649 # messages.append(u"original has an accelerated variable") 650 # if str2.find(error) >= 0: 651 # messages.append(u"translation has an accelerated variable") 652 # if messages: 653 # raise FilterFailure(messages) 654 # return True 655

656 - def variables(self, str1, str2):

657 """checks whether variables of various forms are consistent between the two strings""" 658 messages = [] 659 mismatch1, mismatch2 = [], [] 660 varnames1, varnames2 = [], [] 661 for startmarker, endmarker in self.config.varmatches: 662 varchecker = decoration.getvariables(startmarker, endmarker) 663 if startmarker and endmarker: 664 if isinstance(endmarker, int): 665 redecorate = lambda var: startmarker + var 666 else: 667 redecorate = lambda var: startmarker + var + endmarker 668 elif startmarker: 669 redecorate = lambda var: startmarker + var 670 else: 671 redecorate = lambda var: var 672 vars1 = varchecker(str1) 673 vars2 = varchecker(str2) 674 if vars1 != vars2: 675 # we use counts to compare so we can handle multiple variables 676 vars1, vars2 = [var for var in vars1 if vars1.count(var) > vars2.count(var)], [var for var in vars2 if vars1.count(var) < vars2.count(var)] 677 # filter variable names we've already seen, so they aren't matched by more than one filter... 678 vars1, vars2 = [var for var in vars1 if var not in varnames1], [var for var in vars2 if var not in varnames2] 679 varnames1.extend(vars1) 680 varnames2.extend(vars2) 681 vars1 = map(redecorate, vars1) 682 vars2 = map(redecorate, vars2) 683 mismatch1.extend(vars1) 684 mismatch2.extend(vars2) 685 if mismatch1: 686 messages.append(u"do not translate: %s" % u", ".join(mismatch1)) 687 elif mismatch2: 688 messages.append(u"translation contains variables not in original: %s" % u", ".join(mismatch2)) 689 if messages and mismatch1: 690 raise SeriousFilterFailure(messages) 691 elif messages: 692 raise FilterFailure(messages) 693 return True

694

695 - def functions(self, str1, str2):

696 """checks that function names are not translated""" 697 return helpers.funcmatch(str1, str2, decoration.getfunctions, self.config.punctuation)

698

699 - def emails(self, str1, str2):

700 """checks that emails are not translated""" 701 return helpers.funcmatch(str1, str2, decoration.getemails)

702

703 - def urls(self, str1, str2):

704 """checks that URLs are not translated""" 705 return helpers.funcmatch(str1, str2, decoration.geturls)

706

707 - def numbers(self, str1, str2):

708 """checks whether numbers of various forms are consistent between the two strings""" 709 return helpers.countsmatch(str1, str2, decoration.getnumbers(str1))

710

711 - def startwhitespace(self, str1, str2):

712 """checks whether whitespace at the beginning of the strings matches""" 713 return helpers.funcmatch(str1, str2, decoration.spacestart)

714

715 - def endwhitespace(self, str1, str2):

716 """checks whether whitespace at the end of the strings matches""" 717 str1 = self.config.lang.punctranslate(str1) 718 return helpers.funcmatch(str1, str2, decoration.spaceend)

719

720 - def startpunc(self, str1, str2):

721 """checks whether punctuation at the beginning of the strings match""" 722 str1 = self.filterxml(self.filteraccelerators(self.filtervariables(self.filterwordswithpunctuation(str1)))) 723 str1 = self.config.lang.punctranslate(str1) 724 str2 = self.filterxml(self.filteraccelerators(self.filtervariables(self.filterwordswithpunctuation(str2)))) 725 return helpers.funcmatch(str1, str2, decoration.puncstart, self.config.punctuation)

726

727 - def endpunc(self, str1, str2):

728 """checks whether punctuation at the end of the strings match""" 729 str1 = self.filtervariables(str1) 730 str1 = self.config.lang.punctranslate(str1) 731 str2 = self.filtervariables(str2) 732 str1 = str1.rstrip() 733 str2 = str2.rstrip() 734 return helpers.funcmatch(str1, str2, decoration.puncend, self.config.endpunctuation + u":")

735

736 - def purepunc(self, str1, str2):

737 """checks that strings that are purely punctuation are not changed""" 738 # this test is a subset of startandend 739 if (decoration.ispurepunctuation(str1)): 740 return str1 == str2 741 else: 742 return not decoration.ispurepunctuation(str2)

743

744 - def brackets(self, str1, str2):

745 """checks that the number of brackets in both strings match""" 746 str1 = self.filtervariables(str1) 747 str2 = self.filtervariables(str2) 748 messages = [] 749 missing = [] 750 extra = [] 751 for bracket in (u"[", u"]", u"{", u"}", u"(", u")"): 752 count1 = str1.count(bracket) 753 count2 = str2.count(bracket) 754 if count2 < count1: 755 missing.append(u"'%s'" % bracket) 756 elif count2 > count1: 757 extra.append(u"'%s'" % bracket) 758 if missing: 759 messages.append(u"translation is missing %s" % u", ".join(missing)) 760 if extra: 761 messages.append(u"translation has extra %s" % u", ".join(extra)) 762 if messages: 763 raise FilterFailure(messages) 764 return True

765

766 - def sentencecount(self, str1, str2):

767 """checks that the number of sentences in both strings match""" 768 str1 = self.filteraccelerators(str1) 769 str2 = self.filteraccelerators(str2) 770 sentences1 = len(self.config.sourcelang.sentences(str1)) 771 sentences2 = len(self.config.lang.sentences(str2)) 772 if not sentences1 == sentences2: 773 raise FilterFailure(u"The number of sentences differ: %d versus %d" % (sentences1, sentences2)) 774 return True

775

776 - def options(self, str1, str2):

777 """checks that options are not translated""" 778 str1 = self.filtervariables(str1) 779 for word1 in str1.split(): 780 if word1 != u"--" and word1.startswith(u"--") and word1[-1].isalnum(): 781 parts = word1.split(u"=") 782 if not parts[0] in str2: 783 raise FilterFailure(u"The option %s does not occur or is translated in the translation." % parts[0]) 784 if len(parts) > 1 and parts[1] in str2: 785 raise FilterFailure(u"The parameter %(param)s in option %(option)s is not translated." % {"param": parts[1], "option": parts[0]}) 786 return True

787

788 - def startcaps(self, str1, str2):

789 """checks that the message starts with the correct capitalisation""" 790 str1 = self.filteraccelerators(str1) 791 str2 = self.filteraccelerators(str2) 792 if len(str1) > 1 and len(str2) > 1: 793 return self.config.sourcelang.capsstart(str1) == self.config.lang.capsstart(str2) 794 if len(str1) == 0 and len(str2) == 0: 795 return True 796 if len(str1) == 0 or len(str2) == 0: 797 return False 798 return True

799

800 - def simplecaps(self, str1, str2):

801 """checks the capitalisation of two strings isn't wildly different""" 802 str1 = self.removevariables(str1) 803 str2 = self.removevariables(str2) 804 # TODO: review this. The 'I' is specific to English, so it probably serves 805 # no purpose to get sourcelang.sentenceend 806 str1 = re.sub(u"[^%s]( I )" % self.config.sourcelang.sentenceend, u" i ", str1) 807 capitals1 = helpers.filtercount(str1, unicode.isupper) 808 capitals2 = helpers.filtercount(str2, unicode.isupper) 809 alpha1 = helpers.filtercount(str1, unicode.isalpha) 810 alpha2 = helpers.filtercount(str2, unicode.isalpha) 811 # Capture the all caps case 812 if capitals1 == alpha1: 813 return capitals2 == alpha2 814 # some heuristic tests to try and see that the style of capitals is vaguely the same 815 if capitals1 == 0 or capitals1 == 1: 816 return capitals2 == capitals1 817 elif capitals1 < len(str1) / 10: 818 return capitals2 <= len(str2) / 8 819 elif len(str1) < 10: 820 return abs(capitals1 - capitals2) < 3 821 elif capitals1 > len(str1) * 6 / 10: 822 return capitals2 > len(str2) * 6 / 10 823 else: 824 return abs(capitals1 - capitals2) < (len(str1) + len(str2)) / 6

825

826 - def acronyms(self, str1, str2):

827 """checks that acronyms that appear are unchanged""" 828 acronyms = [] 829 allowed = [] 830 for startmatch, endmatch in self.config.varmatches: 831 allowed += decoration.getvariables(startmatch, endmatch)(str1) 832 allowed += self.config.musttranslatewords.keys() 833 str1 = self.filteraccelerators(self.filtervariables(str1)) 834 iter = self.config.lang.word_iter(str1) 835 str2 = self.filteraccelerators(self.filtervariables(str2)) 836 #TODO: strip XML? - should provide better error messsages 837 # see mail/chrome/messanger/smime.properties.po 838 #TODO: consider limiting the word length for recognising acronyms to 839 #something like 5/6 characters 840 for word in iter: 841 if word.isupper() and len(word) > 1 and word not in allowed: 842 if str2.find(word) == -1: 843 acronyms.append(word) 844 if acronyms: 845 raise FilterFailure(u"acronyms should not be translated: " + u", ".join(acronyms)) 846 return True

847

848 - def doublewords(self, str1, str2):

849 """checks for repeated words in the translation""" 850 lastword = "" 851 without_newlines = "\n".join(str2.split("\n")) 852 words = self.filteraccelerators(self.removevariables(without_newlines)).replace(u".", u"").lower().split() 853 for word in words: 854 if word == lastword and word not in self.config.lang.validdoublewords: 855 raise FilterFailure(u"The word '%s' is repeated" % word) 856 lastword = word 857 return True

858

859 - def notranslatewords(self, str1, str2):

860 """checks that words configured as untranslatable appear in the translation too""" 861 if not self.config.notranslatewords: 862 return True 863 str1 = self.filtervariables(str1) 864 str2 = self.filtervariables(str2) 865 #The above is full of strange quotes and things in utf-8 encoding. 866 #single apostrophe perhaps problematic in words like "doesn't" 867 for seperator in self.config.punctuation: 868 str1 = str1.replace(seperator, u" ") 869 str2 = str2.replace(seperator, u" ") 870 words1 = self.filteraccelerators(str1).split() 871 words2 = self.filteraccelerators(str2).split() 872 stopwords = [word for word in words1 if word in self.config.notranslatewords and word not in words2] 873 if stopwords: 874 raise FilterFailure(u"do not translate: %s" % (u", ".join(stopwords))) 875 return True

876

877 - def musttranslatewords(self, str1, str2):

878 """checks that words configured as definitely translatable don't appear in 879 the translation""" 880 if not self.config.musttranslatewords: 881 return True 882 str1 = self.removevariables(str1) 883 str2 = self.removevariables(str2) 884 #The above is full of strange quotes and things in utf-8 encoding. 885 #single apostrophe perhaps problematic in words like "doesn't" 886 for seperator in self.config.punctuation: 887 str1 = str1.replace(seperator, u" ") 888 str2 = str2.replace(seperator, u" ") 889 words1 = self.filteraccelerators(str1).split() 890 words2 = self.filteraccelerators(str2).split() 891 stopwords = [word for word in words1 if word in self.config.musttranslatewords and word in words2] 892 if stopwords: 893 raise FilterFailure(u"please translate: %s" % (u", ".join(stopwords))) 894 return True

895

896 - def validchars(self, str1, str2):

897 """checks that only characters specified as valid appear in the translation""" 898 if not self.config.validcharsmap: 899 return True 900 invalid1 = str1.translate(self.config.validcharsmap) 901 invalid2 = str2.translate(self.config.validcharsmap) 902 invalidchars = [u"'%s' (\\u%04x)" % (invalidchar, ord(invalidchar)) for invalidchar in invalid2 if invalidchar not in invalid1] 903 if invalidchars: 904 raise FilterFailure(u"invalid chars: %s" % (u", ".join(invalidchars))) 905 return True

906

907 - def filepaths(self, str1, str2):

908 """checks that file paths have not been translated""" 909 for word1 in self.filteraccelerators(str1).split(): 910 if word1.startswith(u"/"): 911 if not helpers.countsmatch(str1, str2, (word1,)): 912 return False 913 return True

914

915 - def xmltags(self, str1, str2):

916 """checks that XML/HTML tags have not been translated""" 917 tags1 = tag_re.findall(str1) 918 if len(tags1) > 0: 919 if (len(tags1[0]) == len(str1)) and not u"=" in tags1[0]: 920 return True 921 tags2 = tag_re.findall(str2) 922 properties1 = tagproperties(tags1, self.config.ignoretags) 923 properties2 = tagproperties(tags2, self.config.ignoretags) 924 filtered1 = [] 925 filtered2 = [] 926 for property1 in properties1: 927 filtered1 += [intuplelist(property1, self.config.canchangetags)] 928 for property2 in properties2: 929 filtered2 += [intuplelist(property2, self.config.canchangetags)] 930 931 #TODO: consider the consequences of different ordering of attributes/tags 932 if filtered1 != filtered2: 933 return False 934 else: 935 # No tags in str1, let's just check that none were added in str2. This 936 # might be useful for fuzzy strings wrongly unfuzzied, for example. 937 tags2 = tag_re.findall(str2) 938 if len(tags2) > 0: 939 return False 940 return True

941

942 - def kdecomments(self, str1, str2):

943 """checks to ensure that no KDE style comments appear in the translation""" 944 return str2.find(u"\n_:") == -1 and not str2.startswith(u"_:")

945

946 - def compendiumconflicts(self, str1, str2):

947 """checks for Gettext compendium conflicts (#-#-#-#-#)""" 948 return str2.find(u"#-#-#-#-#") == -1

949

950 - def simpleplurals(self, str1, str2):

951 """checks for English style plural(s) for you to review""" 952 def numberofpatterns(string, patterns): 953 number = 0 954 for pattern in patterns: 955 number += len(re.findall(pattern, string)) 956 return number

957 958 sourcepatterns = ["$s$"] 959 targetpatterns = ["$s$"] 960 sourcecount = numberofpatterns(str1, sourcepatterns) 961 targetcount = numberofpatterns(str2, targetpatterns) 962 if self.config.lang.nplurals == 1: 963 return not targetcount 964 return sourcecount == targetcount

965

966 - def spellcheck(self, str1, str2):

967 """checks words that don't pass a spell check""" 968 if not self.config.targetlanguage: 969 return True 970 if not spelling.available: 971 return True 972 # TODO: filterxml? 973 str1 = self.filteraccelerators_by_list(self.filtervariables(str1), self.config.sourcelang.validaccel) 974 str2 = self.filteraccelerators_by_list(self.filtervariables(str2), self.config.lang.validaccel) 975 ignore1 = [] 976 messages = [] 977 for word, index, suggestions in spelling.check(str1, lang="en"): 978 ignore1.append(word) 979 for word, index, suggestions in spelling.check(str2, lang=self.config.targetlanguage): 980 if word in self.config.notranslatewords: 981 continue 982 if word in ignore1: 983 continue 984 # hack to ignore hyphenisation rules 985 if word in suggestions: 986 continue 987 messages.append(u"check spelling of %s (could be %s)" % (word, u" / ".join(suggestions[:5]))) 988 if messages: 989 raise FilterFailure(messages) 990 return True

991

992 - def credits(self, str1, str2):

993 """checks for messages containing translation credits instead of normal translations.""" 994 return not str1 in self.config.credit_sources

995 996 # If the precondition filter is run and fails then the other tests listed are ignored 997 preconditions = {"untranslated": ("simplecaps", "variables", "startcaps", 998 "accelerators", "brackets", "endpunc", 999 "acronyms", "xmltags", "startpunc", 1000 "endwhitespace", "startwhitespace", 1001 "escapes", "doublequoting", "singlequoting", 1002 "filepaths", "purepunc", "doublespacing", 1003 "sentencecount", "numbers", "isfuzzy", 1004 "isreview", "notranslatewords", "musttranslatewords", 1005 "emails", "simpleplurals", "urls", "printf", 1006 "tabs", "newlines", "functions", "options", 1007 "blank", "nplurals", "gconf"), 1008 "blank": ("simplecaps", "variables", "startcaps", 1009 "accelerators", "brackets", "endpunc", 1010 "acronyms", "xmltags", "startpunc", 1011 "endwhitespace", "startwhitespace", 1012 "escapes", "doublequoting", "singlequoting", 1013 "filepaths", "purepunc", "doublespacing", 1014 "sentencecount", "numbers", "isfuzzy", 1015 "isreview", "notranslatewords", "musttranslatewords", 1016 "emails", "simpleplurals", "urls", "printf", 1017 "tabs", "newlines", "functions", "options", 1018 "gconf"), 1019 "credits": ("simplecaps", "variables", "startcaps", 1020 "accelerators", "brackets", "endpunc", 1021 "acronyms", "xmltags", "startpunc", 1022 "escapes", "doublequoting", "singlequoting", 1023 "filepaths", "doublespacing", 1024 "sentencecount", "numbers", 1025 "emails", "simpleplurals", "urls", "printf", 1026 "tabs", "newlines", "functions", "options"), 1027 "purepunc": ("startcaps", "options"), 1028 # This is causing some problems since Python 2.6, as 1029 # startcaps is now seen as an important one to always execute 1030 # and could now be done before it is blocked by a failing 1031 # "untranslated" or "blank" test. This is probably happening 1032 # due to slightly different implementation of the internal 1033 # dict handling since Python 2.6. We should never have relied 1034 # on this ordering anyway. 1035 #"startcaps": ("simplecaps",), 1036 "endwhitespace": ("endpunc",), 1037 "startwhitespace":("startpunc",), 1038 "unchanged": ("doublewords",), 1039 "compendiumconflicts": ("accelerators", "brackets", "escapes", 1040 "numbers", "startpunc", "long", "variables", 1041 "startcaps", "sentencecount", "simplecaps", 1042 "doublespacing", "endpunc", "xmltags", 1043 "startwhitespace", "endwhitespace", 1044 "singlequoting", "doublequoting", 1045 "filepaths", "purepunc", "doublewords", "printf") } 1046 1047 # code to actually run the tests (use unittest?) 1048 1049 openofficeconfig = CheckerConfig( 1050 accelmarkers = ["~"], 1051 varmatches = [("&", ";"), ("%", "%"), ("%", None), ("%", 0), ("$(", ")"), ("$", "$"), ("${", "}"), ("#", "#"), ("#", 1), ("#", 0), ("($", ")"), ("$[", "]"), ("[", "]"), ("$", None)], 1052 ignoretags = [("alt", "xml-lang", None), ("ahelp", "visibility", "visible"), ("img", "width", None), ("img", "height", None)], 1053 canchangetags = [("link", "name", None)] 1054 ) 1055

1056 -class OpenOfficeChecker(StandardChecker):

1057 - def __init__(self, **kwargs):

1058 checkerconfig = kwargs.get("checkerconfig", None) 1059 if checkerconfig is None: 1060 checkerconfig = CheckerConfig() 1061 kwargs["checkerconfig"] = checkerconfig 1062 checkerconfig.update(openofficeconfig) 1063 StandardChecker.__init__(self, **kwargs)

1064 1065 mozillaconfig = CheckerConfig( 1066 accelmarkers = ["&"], 1067 varmatches = [("&", ";"), ("%", "%"), ("%", 1), ("$", "$"), ("$", None), ("#", 1), ("${", "}"), ("$(^", ")")], 1068 criticaltests = ["accelerators"] 1069 ) 1070

1071 -class MozillaChecker(StandardChecker):

1072 - def __init__(self, **kwargs):

1073 checkerconfig = kwargs.get("checkerconfig", None) 1074 if checkerconfig is None: 1075 checkerconfig = CheckerConfig() 1076 kwargs["checkerconfig"] = checkerconfig 1077 checkerconfig.update(mozillaconfig) 1078 StandardChecker.__init__(self, **kwargs)

1079

1080 - def credits(self, str1, str2):

1081 """checks for messages containing translation credits instead of normal translations.""" 1082 for location in self.locations: 1083 if location in ['MOZ_LANGPACK_CONTRIBUTORS', 'credit.translation']: 1084 return False 1085 return True

1086 1087 drupalconfig = CheckerConfig( 1088 varmatches = [("%", None), ("@", None), ("!", None)], 1089 ) 1090

1091 -class DrupalChecker(StandardChecker):

1092 - def __init__(self, **kwargs):

1093 checkerconfig = kwargs.get("checkerconfig", None) 1094 if checkerconfig is None: 1095 checkerconfig = CheckerConfig() 1096 kwargs["checkerconfig"] = checkerconfig 1097 checkerconfig.update(drupalconfig) 1098 StandardChecker.__init__(self, **kwargs)

1099 1100 gnomeconfig = CheckerConfig( 1101 accelmarkers = ["_"], 1102 varmatches = [("%", 1), ("$(", ")")], 1103 credit_sources = [u"translator-credits"] 1104 ) 1105

1106 -class GnomeChecker(StandardChecker):

1107 - def __init__(self, **kwargs):

1108 checkerconfig = kwargs.get("checkerconfig", None) 1109 if checkerconfig is None: 1110 checkerconfig = CheckerConfig() 1111 kwargs["checkerconfig"] = checkerconfig 1112 checkerconfig.update(gnomeconfig) 1113 StandardChecker.__init__(self, **kwargs)

1114

1115 - def gconf(self, str1, str2):

1116 """Checks if we have any gconf config settings translated.""" 1117 for location in self.locations: 1118 if location.find('schemas.in') != -1: 1119 gconf_attributes = gconf_attribute_re.findall(str1) 1120 #stopwords = [word for word in words1 if word in self.config.notranslatewords and word not in words2] 1121 stopwords = [word for word in gconf_attributes if word[1:-1] not in str2] 1122 if stopwords: 1123 raise FilterFailure(u"do not translate gconf attribute: %s" % (u", ".join(stopwords))) 1124 return True

1125 1126 kdeconfig = CheckerConfig( 1127 accelmarkers = ["&"], 1128 varmatches = [("%", 1)], 1129 credit_sources = [u"Your names", u"Your emails", u"ROLES_OF_TRANSLATORS"] 1130 ) 1131

1132 -class KdeChecker(StandardChecker):

1133 - def __init__(self, **kwargs):

1134 # TODO allow setup of KDE plural and translator comments so that they do 1135 # not create false postives 1136 checkerconfig = kwargs.get("checkerconfig", None) 1137 if checkerconfig is None: 1138 checkerconfig = CheckerConfig() 1139 kwargs["checkerconfig"] = checkerconfig 1140 checkerconfig.update(kdeconfig) 1141 StandardChecker.__init__(self, **kwargs)

1142 1143 cclicenseconfig = CheckerConfig(varmatches = [("@", "@")])

1144 -class CCLicenseChecker(StandardChecker):

1145 - def __init__(self, **kwargs):

1146 checkerconfig = kwargs.get("checkerconfig", None) 1147 if checkerconfig is None: 1148 checkerconfig = CheckerConfig() 1149 kwargs["checkerconfig"] = checkerconfig 1150 checkerconfig.update(cclicenseconfig) 1151 StandardChecker.__init__(self, **kwargs)

1152 1153 projectcheckers = { 1154 "openoffice": OpenOfficeChecker, 1155 "mozilla": MozillaChecker, 1156 "kde": KdeChecker, 1157 "wx": KdeChecker, 1158 "gnome": GnomeChecker, 1159 "creativecommons": CCLicenseChecker, 1160 "drupal": DrupalChecker, 1161 } 1162 1163

1164 -class StandardUnitChecker(UnitChecker):

1165 """The standard checks for common checks on translation units."""

1166 - def isfuzzy(self, unit):

1167 """Check if the unit has been marked fuzzy.""" 1168 return not unit.isfuzzy()

1169

1170 - def isreview(self, unit):

1171 """Check if the unit has been marked review.""" 1172 return not unit.isreview()

1173

1174 - def nplurals(self, unit):

1175 """Checks for the correct number of noun forms for plural translations.""" 1176 if unit.hasplural(): 1177 # if we don't have a valid nplurals value, don't run the test 1178 nplurals = self.config.lang.nplurals 1179 if nplurals > 0: 1180 return len(unit.target.strings) == nplurals 1181 return True

1182

1183 - def hassuggestion(self, unit):

1184 """Checks if there is at least one suggested translation for this unit.""" 1185 self.suggestion_store = getattr(self, 'suggestion_store', None) 1186 suggestions = [] 1187 if self.suggestion_store: 1188 suggestions = self.suggestion_store.findunits(unit.source) 1189 elif xliff and isinstance(unit, xliff.xliffunit): 1190 # TODO: we probably want to filter them somehow 1191 suggestions = unit.getalttrans() 1192 return not bool(suggestions)

1193 1194

1195 -def runtests(str1, str2, ignorelist=()):

1196 """verifies that the tests pass for a pair of strings""" 1197 from translate.storage import base 1198 str1 = data.normalized_unicode(str1) 1199 str2 = data.normalized_unicode(str2) 1200 unit = base.TranslationUnit(str1) 1201 unit.target = str2 1202 checker = StandardChecker(excludefilters=ignorelist) 1203 failures = checker.run_filters(unit) 1204 for test in failures: 1205 print "failure: %s: %s\n %r\n %r" % (test, failures[test], str1, str2) 1206 return failures

1207

1208 -def batchruntests(pairs):

1209 """runs test on a batch of string pairs""" 1210 passed, numpairs = 0, len(pairs) 1211 for str1, str2 in pairs: 1212 if runtests(str1, str2): 1213 passed += 1 1214 print 1215 print "total: %d/%d pairs passed" % (passed, numpairs)

1216 1217 if __name__ == '__main__': 1218 testset = [(r"simple", r"somple"), 1219 (r"\this equals \that", r"does \this equal \that?"), 1220 (r"this \'equals\' that", r"this 'equals' that"), 1221 (r" start and end! they must match.", r"start and end! they must match."), 1222 (r"check for matching %variables marked like %this", r"%this %variable is marked"), 1223 (r"check for mismatching %variables marked like %this", r"%that %variable is marked"), 1224 (r"check for mismatching %variables% too", r"how many %variable% are marked"), 1225 (r"%% %%", r"%%"), 1226 (r"Row: %1, Column: %2", r"Mothalo: %1, Kholomo: %2"), 1227 (r"simple lowercase", r"it is all lowercase"), 1228 (r"simple lowercase", r"It Is All Lowercase"), 1229 (r"Simple First Letter Capitals", r"First Letters"), 1230 (r"SIMPLE CAPITALS", r"First Letters"), 1231 (r"SIMPLE CAPITALS", r"ALL CAPITALS"), 1232 (r"forgot to translate", r" ") 1233 ] 1234 batchruntests(testset) 1235

Source Code for Module translate.filters.checks