src/cm/utils/string.py
author raph
Mon, 25 Jan 2010 11:34:22 +0100
changeset 119 5e8dda1b7631
permissions -rw-r--r--
recover when tidy trashes: try markdown anyway
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
119
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     1
import chardet
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     2
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     3
def to_unicode(input):
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     4
    if type(input) == str:
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     5
        res = None
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     6
        for encoding in [chardet.detect(input)['encoding'], 'utf8', 'latin1']:
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     7
            try:
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     8
                res = unicode(input, encoding)
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     9
                break;
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
    10
            except UnicodeDecodeError:
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
    11
                pass
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
    12
        if not res:
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
    13
            raise Exception('UnicodeDecodeError: could not decode')
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
    14
        return res
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
    15
    return input