src/cm/utils/string_utils.py
author raph
Sat, 06 Feb 2010 18:04:01 +0100
changeset 151 150cb6e76f30
parent 149 0f2c5744b39b
child 175 4f072edc51a1
permissions -rw-r--r--
add permission check on text compare
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
119
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     1
import chardet
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     2
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     3
def to_unicode(input):
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     4
    if type(input) == str:
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     5
        res = None
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     6
        for encoding in [chardet.detect(input)['encoding'], 'utf8', 'latin1']:
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     7
            try:
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     8
                res = unicode(input, encoding)
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
     9
                break;
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
    10
            except UnicodeDecodeError:
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
    11
                pass
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
    12
        if not res:
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
    13
            raise Exception('UnicodeDecodeError: could not decode')
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
    14
        return res
5e8dda1b7631 recover when tidy trashes: try markdown anyway
raph
parents:
diff changeset
    15
    return input