Package translate :: Package convert :: Module csv2po

[hide private]
[frames] | no frames]

# Source Code for Module translate.convert.csv2po

      1  #!/usr/bin/env python
      2  # -*- coding: utf-8 -*-
      3  #
      4  # Copyright 2003-2006 Zuza Software Foundation
      5  #
      6  # This file is part of translate.
      7  #
      8  # translate is free software; you can redistribute it and/or modify
      9  # it under the terms of the GNU General Public License as published by
     10  # the Free Software Foundation; either version 2 of the License, or
     11  # (at your option) any later version.
     12  #
     13  # translate is distributed in the hope that it will be useful,
     14  # but WITHOUT ANY WARRANTY; without even the implied warranty of
     15  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     16  # GNU General Public License for more details.
     17  #
     18  # You should have received a copy of the GNU General Public License
     19  # along with translate; if not, write to the Free Software
     20  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
     21
     22  """convert Comma-Separated Value (.csv) files to Gettext PO localization files
     23
     24  See: http://translate.sourceforge.net/wiki/toolkit/csv2po for examples and
     25  usage instructions
     26  """
     27
     28  import sys
     29
     30  from translate.misc import sparse
     31  from translate.storage import po
     32  from translate.storage import csvl10n
     33
     34
     35 -def replacestrings(source, *pairs):
     36      for orig, new in pairs:
     37          source = source.replace(orig, new)
     38      return source
     39
     40
     41 -def quotecsvstr(source):
     42      return '"' + replacestrings(source, ('\\"', '"'), ('"', '\\"'), ("\\\\'", "\\'"), ('\\\\n', '\\n')) + '"'
     43
     44
     45 -def simplify(string):
     46      return filter(type(string).isalnum, string)
     47      tokens = sparse.SimpleParser().tokenize(string)
     48      return " ".join(tokens)
     49
     50
     51 -class csv2po:
     52      """a class that takes translations from a .csv file and puts them in a .po file"""
     53
     54 -    def __init__(self, templatepo=None, charset=None, duplicatestyle="keep"):
     55          """construct the converter..."""
     56          self.pofile = templatepo
     57          self.charset = charset
     58          self.duplicatestyle = duplicatestyle
     59          if self.pofile is not None:
     60              self.unmatched = 0
     61              self.makeindex()
     62
     63 -    def makeindex(self):
     64          """makes indexes required for searching..."""
     65          self.commentindex = {}
     66          self.sourceindex = {}
     67          self.simpleindex = {}
     68          self.duplicatecomments = []
     69          for pounit in self.pofile.units:
     70              joinedcomment = " ".join(pounit.getlocations())
     71              source = pounit.source
     72              # the definitive way to match is by source comment (joinedcomment)
     73              if joinedcomment in self.commentindex:
     74                  # unless more than one thing matches...
     75                  self.duplicatecomments.append(joinedcomment)
     76              else:
     77                  self.commentindex[joinedcomment] = pounit
     78              # do simpler matching in case things have been mangled...
     79              simpleid = simplify(source)
     80              # but check for duplicates
     81              if simpleid in self.simpleindex and not (source in self.sourceindex):
     82                  # keep a list of them...
     83                  self.simpleindex[simpleid].append(pounit)
     84              else:
     85                  self.simpleindex[simpleid] = [pounit]
     86              # also match by standard msgid
     87              self.sourceindex[source] = pounit
     88          for comment in self.duplicatecomments:
     89              if comment in self.commentindex:
     90                  del self.commentindex[comment]
     91
     92 -    def convertunit(self, csvunit):
     93          """converts csv unit to po unit"""
     94          pounit = po.pounit(encoding="UTF-8")
     95          if csvunit.location:
     96              pounit.addlocation(csvunit.location)
     97          pounit.source = csvunit.source
     98          pounit.target = csvunit.target
     99          return pounit
    100
    101 -    def handlecsvunit(self, csvunit):
    102          """handles reintegrating a csv unit into the .po file"""
    103          if len(csvunit.location.strip()) > 0 and csvunit.location in self.commentindex:
    104              pounit = self.commentindex[csvunit.location]
    105          elif csvunit.source in self.sourceindex:
    106              pounit = self.sourceindex[csvunit.source]
    107          elif simplify(csvunit.source) in self.simpleindex:
    108              thepolist = self.simpleindex[simplify(csvunit.source)]
    109              if len(thepolist) > 1:
    110                  csvfilename = getattr(self.csvfile, "filename", "(unknown)")
    111                  matches = "\n  ".join(["possible match: " + pounit.source for pounit in thepolist])
    112                  print >> sys.stderr, "%s - csv entry not found in pofile, multiple matches found:\n  location\t%s\n  original\t%s\n  translation\t%s\n  %s" % (csvfilename, csvunit.location, csvunit.source, csvunit.target, matches)
    113                  self.unmatched += 1
    114                  return
    115              pounit = thepolist[0]
    116          else:
    117              csvfilename = getattr(self.csvfile, "filename", "(unknown)")
    118              print >> sys.stderr, "%s - csv entry not found in pofile:\n  location\t%s\n  original\t%s\n  translation\t%s" % (csvfilename, csvunit.location, csvunit.source, csvunit.target)
    119              self.unmatched += 1
    120              return
    121          if pounit.hasplural():
    122              # we need to work out whether we matched the singular or the plural
    123              singularid = pounit.source.strings[0]
    124              pluralid = pounit.source.strings[1]
    125              if csvunit.source == singularid:
    126                  pounit.msgstr[0] = csvunit.target
    127              elif csvunit.source == pluralid:
    128                  pounit.msgstr[1] = csvunit.target
    129              elif simplify(csvunit.source) == simplify(singularid):
    130                  pounit.msgstr[0] = csvunit.target
    131              elif simplify(csvunit.source) == simplify(pluralid):
    132                  pounit.msgstr[1] = csvunit.target
    133              else:
    134                  print >> sys.stderr, "couldn't work out singular or plural: %r, %r, %r" %  \
    135                      (csvunit.source, singularid, pluralid)
    136                  self.unmatched += 1
    137                  return
    138          else:
    139              pounit.target = csvunit.target
    140
    141 -    def convertstore(self, thecsvfile):
    142          """converts a csvfile to a pofile, and returns it. uses templatepo if given at construction"""
    143          self.csvfile = thecsvfile
    144          if self.pofile is None:
    145              self.pofile = po.pofile()
    146              mergemode = False
    147          else:
    148              mergemode = True
    149          if self.pofile.units and self.pofile.units[0].isheader():
    150              targetheader = self.pofile.units[0]
    151              self.pofile.updateheader(content_type="text/plain; charset=UTF-8", content_transfer_encoding="8bit")
    152          else:
    153              targetheader = self.pofile.makeheader(charset="UTF-8", encoding="8bit")
    154          targetheader.addnote("extracted from %s" % self.csvfile.filename, "developer")
    155          mightbeheader = True
    156          for csvunit in self.csvfile.units:
    157              if self.charset is not None:
    158                  csvunit.source = csvunit.source.decode(self.charset)
    159                  csvunit.target = csvunit.target.decode(self.charset)
    160              if mightbeheader:
    161                  # ignore typical header strings...
    162                  mightbeheader = False
    163                  if [item.strip().lower() for item in csvunit.location, csvunit.source, csvunit.target] == \
    164                          ["location", "source", "target"]:
    165                      continue
    166                  if len(csvunit.location.strip()) == 0 and csvunit.source.find("Content-Type:") != -1:
    167                      continue
    168              if mergemode:
    169                  self.handlecsvunit(csvunit)
    170              else:
    171                  pounit = self.convertunit(csvunit)
    172                  self.pofile.addunit(pounit)
    173          self.pofile.removeduplicates(self.duplicatestyle)
    174          return self.pofile
    175
    176
    177 -def convertcsv(inputfile, outputfile, templatefile, charset=None, columnorder=None, duplicatestyle="msgctxt"):
    178      """reads in inputfile using csvl10n, converts using csv2po, writes to outputfile"""
    179      inputstore = csvl10n.csvfile(inputfile, fieldnames=columnorder)
    180      if templatefile is None:
    181          convertor = csv2po(charset=charset, duplicatestyle=duplicatestyle)
    182      else:
    183          templatestore = po.pofile(templatefile)
    184          convertor = csv2po(templatestore, charset=charset, duplicatestyle=duplicatestyle)
    185      outputstore = convertor.convertstore(inputstore)
    186      if outputstore.isempty():
    187          return 0
    188      outputfile.write(str(outputstore))
    189      return 1
    190
    191
    192 -def main(argv=None):
    193      from translate.convert import convert
    194      formats = {("csv", "po"): ("po", convertcsv), ("csv", "pot"): ("po", convertcsv),
    195              ("csv", None): ("po", convertcsv)}
    196      parser = convert.ConvertOptionParser(formats, usetemplates=True, description=__doc__)
    197      parser.add_option("", "--charset", dest="charset", default=None,
    198          help="set charset to decode from csv files", metavar="CHARSET")
    199      parser.add_option("", "--columnorder", dest="columnorder", default=None,
    200          help="specify the order and position of columns (location,source,target)")
    201      parser.add_duplicates_option()
    202      parser.passthrough.append("charset")
    203      parser.passthrough.append("columnorder")
    204      parser.run(argv)
    205
    206
    207  if __name__ == '__main__':
    208      main()
    209