src/cm/utils/string.py
author raph
Fri, 05 Feb 2010 15:15:46 +0100
changeset 148 03106bfa4845
parent 119 5e8dda1b7631
permissions -rw-r--r--
fix version compare order / i18n update (fr)
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
119
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     1
import chardet
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     2
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     3
def to_unicode(input):
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     4
    if type(input) == str:
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     5
        res = None
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     6
        for encoding in [chardet.detect(input)['encoding'], 'utf8', 'latin1']:
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     7
            try:
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     8
                res = unicode(input, encoding)
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     9
                break;
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
    10
            except UnicodeDecodeError:
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
    11
                pass
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
    12
        if not res:
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
    13
            raise Exception('UnicodeDecodeError: could not decode')
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
    14
        return res
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
    15
    return input