1 <?php

     2 // $Id: unicode.inc,v 1.29 2007/12/28 12:02:50 dries Exp $

     3 

     4 /**

     5  * Indicates an error during check for PHP unicode support.

     6  */

     7 define('UNICODE_ERROR', -1);

     8 

     9 /**

    10  * Indicates that standard PHP (emulated) unicode support is being used.

    11  */

    12 define('UNICODE_SINGLEBYTE', 0);

    13 

    14 /**

    15  * Indicates that full unicode support with the PHP mbstring extension is being

    16  * used.

    17  */

    18 define('UNICODE_MULTIBYTE', 1);

    19 

    20 /**

    21  * Wrapper around _unicode_check().

    22  */

    23 function unicode_check() {

    24   list($GLOBALS['multibyte']) = _unicode_check();

    25 }

    26 

    27 /**

    28  * Perform checks about Unicode support in PHP, and set the right settings if

    29  * needed.

    30  *

    31  * Because Drupal needs to be able to handle text in various encodings, we do

    32  * not support mbstring function overloading. HTTP input/output conversion must

    33  * be disabled for similar reasons.

    34  *

    35  * @param $errors

    36  *   Whether to report any fatal errors with form_set_error().

    37  */

    38 function _unicode_check() {

    39   // Ensure translations don't break at install time

    40   $t = get_t();

    41 

    42   // Set the standard C locale to ensure consistent, ASCII-only string handling.

    43   setlocale(LC_CTYPE, 'C');

    44 

    45   // Check for outdated PCRE library

    46   // Note: we check if U+E2 is in the range U+E0 - U+E1. This test returns TRUE on old PCRE versions.

    47   if (preg_match('/[à-á]/u', 'â')) {

    48     return array(UNICODE_ERROR, $t('The PCRE library in your PHP installation is outdated. This will cause problems when handling Unicode text. If you are running PHP 4.3.3 or higher, make sure you are using the PCRE library supplied by PHP. Please refer to the <a href="@url">PHP PCRE documentation</a> for more information.', array('@url' => 'http://www.php.net/pcre')));

    49   }

    50 

    51   // Check for mbstring extension

    52   if (!function_exists('mb_strlen')) {

    53     return array(UNICODE_SINGLEBYTE, $t('Operations on Unicode strings are emulated on a best-effort basis. Install the <a href="@url">PHP mbstring extension</a> for improved Unicode support.', array('@url' => 'http://www.php.net/mbstring')));

    54   }

    55 

    56   // Check mbstring configuration

    57   if (ini_get('mbstring.func_overload') != 0) {

    58     return array(UNICODE_ERROR, $t('Multibyte string function overloading in PHP is active and must be disabled. Check the php.ini <em>mbstring.func_overload</em> setting. Please refer to the <a href="@url">PHP mbstring documentation</a> for more information.', array('@url' => 'http://www.php.net/mbstring')));

    59   }

    60   if (ini_get('mbstring.encoding_translation') != 0) {

    61     return array(UNICODE_ERROR, $t('Multibyte string input conversion in PHP is active and must be disabled. Check the php.ini <em>mbstring.encoding_translation</em> setting. Please refer to the <a href="@url">PHP mbstring documentation</a> for more information.', array('@url' => 'http://www.php.net/mbstring')));

    62   }

    63   if (ini_get('mbstring.http_input') != 'pass') {

    64     return array(UNICODE_ERROR, $t('Multibyte string input conversion in PHP is active and must be disabled. Check the php.ini <em>mbstring.http_input</em> setting. Please refer to the <a href="@url">PHP mbstring documentation</a> for more information.', array('@url' => 'http://www.php.net/mbstring')));

    65   }

    66   if (ini_get('mbstring.http_output') != 'pass') {

    67     return array(UNICODE_ERROR, $t('Multibyte string output conversion in PHP is active and must be disabled. Check the php.ini <em>mbstring.http_output</em> setting. Please refer to the <a href="@url">PHP mbstring documentation</a> for more information.', array('@url' => 'http://www.php.net/mbstring')));

    68   }

    69 

    70   // Set appropriate configuration

    71   mb_internal_encoding('utf-8');

    72   mb_language('uni');

    73   return array(UNICODE_MULTIBYTE, '');

    74 }

    75 

    76 /**

    77  * Return Unicode library status and errors.

    78  */

    79 function unicode_requirements() {

    80   // Ensure translations don't break at install time

    81   $t = get_t();

    82 

    83   $libraries = array(

    84     UNICODE_SINGLEBYTE => $t('Standard PHP'),

    85     UNICODE_MULTIBYTE => $t('PHP Mbstring Extension'),

    86     UNICODE_ERROR => $t('Error'),

    87   );

    88   $severities = array(

    89     UNICODE_SINGLEBYTE => REQUIREMENT_WARNING,

    90     UNICODE_MULTIBYTE => REQUIREMENT_OK,

    91     UNICODE_ERROR => REQUIREMENT_ERROR,

    92   );

    93   list($library, $description) = _unicode_check();

    94 

    95   $requirements['unicode'] = array(

    96     'title' => $t('Unicode library'),

    97     'value' => $libraries[$library],

    98   );

    99   if ($description) {

   100     $requirements['unicode']['description'] = $description;

   101   }

   102 

   103   $requirements['unicode']['severity'] = $severities[$library];

   104 

   105   return $requirements;

   106 }

   107 

   108 /**

   109  * Prepare a new XML parser.

   110  *

   111  * This is a wrapper around xml_parser_create() which extracts the encoding from

   112  * the XML data first and sets the output encoding to UTF-8. This function should

   113  * be used instead of xml_parser_create(), because PHP 4's XML parser doesn't

   114  * check the input encoding itself. "Starting from PHP 5, the input encoding is

   115  * automatically detected, so that the encoding parameter specifies only the

   116  * output encoding."

   117  *

   118  * This is also where unsupported encodings will be converted. Callers should

   119  * take this into account: $data might have been changed after the call.

   120  *

   121  * @param &$data

   122  *   The XML data which will be parsed later.

   123  * @return

   124  *   An XML parser object.

   125  */

   126 function drupal_xml_parser_create(&$data) {

   127   // Default XML encoding is UTF-8

   128   $encoding = 'utf-8';

   129   $bom = FALSE;

   130 

   131   // Check for UTF-8 byte order mark (PHP5's XML parser doesn't handle it).

   132   if (!strncmp($data, "\xEF\xBB\xBF", 3)) {

   133     $bom = TRUE;

   134     $data = substr($data, 3);

   135   }

   136 

   137   // Check for an encoding declaration in the XML prolog if no BOM was found.

   138   if (!$bom && ereg('^<\?xml[^>]+encoding="([^"]+)"', $data, $match)) {

   139     $encoding = $match[1];

   140   }

   141 

   142   // Unsupported encodings are converted here into UTF-8.

   143   $php_supported = array('utf-8', 'iso-8859-1', 'us-ascii');

   144   if (!in_array(strtolower($encoding), $php_supported)) {

   145     $out = drupal_convert_to_utf8($data, $encoding);

   146     if ($out !== FALSE) {

   147       $encoding = 'utf-8';

   148       $data = ereg_replace('^(<\?xml[^>]+encoding)="([^"]+)"', '\\1="utf-8"', $out);

   149     }

   150     else {

   151       watchdog('php', 'Could not convert XML encoding %s to UTF-8.', array('%s' => $encoding), WATCHDOG_WARNING);

   152       return 0;

   153     }

   154   }

   155 

   156   $xml_parser = xml_parser_create($encoding);

   157   xml_parser_set_option($xml_parser, XML_OPTION_TARGET_ENCODING, 'utf-8');

   158   return $xml_parser;

   159 }

   160 

   161 /**

   162  * Convert data to UTF-8

   163  *

   164  * Requires the iconv, GNU recode or mbstring PHP extension.

   165  *

   166  * @param $data

   167  *   The data to be converted.

   168  * @param $encoding

   169  *   The encoding that the data is in

   170  * @return

   171  *   Converted data or FALSE.

   172  */

   173 function drupal_convert_to_utf8($data, $encoding) {

   174   if (function_exists('iconv')) {

   175     $out = @iconv($encoding, 'utf-8', $data);

   176   }

   177   else if (function_exists('mb_convert_encoding')) {

   178     $out = @mb_convert_encoding($data, 'utf-8', $encoding);

   179   }

   180   else if (function_exists('recode_string')) {

   181     $out = @recode_string($encoding .'..utf-8', $data);

   182   }

   183   else {

   184     watchdog('php', 'Unsupported encoding %s. Please install iconv, GNU recode or mbstring for PHP.', array('%s' => $encoding), WATCHDOG_ERROR);

   185     return FALSE;

   186   }

   187 

   188   return $out;

   189 }

   190 

   191 /**

   192  * Truncate a UTF-8-encoded string safely to a number of bytes.

   193  *

   194  * If the end position is in the middle of a UTF-8 sequence, it scans backwards

   195  * until the beginning of the byte sequence.

   196  *

   197  * Use this function whenever you want to chop off a string at an unsure

   198  * location. On the other hand, if you're sure that you're splitting on a

   199  * character boundary (e.g. after using strpos() or similar), you can safely use

   200  * substr() instead.

   201  *

   202  * @param $string

   203  *   The string to truncate.

   204  * @param $len

   205  *   An upper limit on the returned string length.

   206  * @return

   207  *   The truncated string.

   208  */

   209 function drupal_truncate_bytes($string, $len) {

   210   if (strlen($string) <= $len) {

   211     return $string;

   212   }

   213   if ((ord($string[$len]) < 0x80) || (ord($string[$len]) >= 0xC0)) {

   214     return substr($string, 0, $len);

   215   }

   216   while (--$len >= 0 && ord($string[$len]) >= 0x80 && ord($string[$len]) < 0xC0) {};

   217   return substr($string, 0, $len);

   218 }

   219 

   220 /**

   221  * Truncate a UTF-8-encoded string safely to a number of characters.

   222  *

   223  * @param $string

   224  *   The string to truncate.

   225  * @param $len

   226  *   An upper limit on the returned string length.

   227  * @param $wordsafe

   228  *   Flag to truncate at last space within the upper limit. Defaults to FALSE.

   229  * @param $dots

   230  *   Flag to add trailing dots. Defaults to FALSE.

   231  * @return

   232  *   The truncated string.

   233  */

   234 function truncate_utf8($string, $len, $wordsafe = FALSE, $dots = FALSE) {

   235 

   236   if (drupal_strlen($string) <= $len) {

   237     return $string;

   238   }

   239 

   240   if ($dots) {

   241     $len -= 4;

   242   }

   243 

   244   if ($wordsafe) {

   245     $string = drupal_substr($string, 0, $len + 1); // leave one more character

   246     if ($last_space = strrpos($string, ' ')) { // space exists AND is not on position 0

   247       $string = substr($string, 0, $last_space);

   248     }

   249     else {

   250       $string = drupal_substr($string, 0, $len);

   251     }

   252   }

   253   else {

   254     $string = drupal_substr($string, 0, $len);

   255   }

   256 

   257   if ($dots) {

   258     $string .= ' ...';

   259   }

   260 

   261   return $string;

   262 }

   263 

   264 /**

   265  * Encodes MIME/HTTP header values that contain non-ASCII, UTF-8 encoded

   266  * characters.

   267  *

   268  * For example, mime_header_encode('tést.txt') returns "=?UTF-8?B?dMOpc3QudHh0?=".

   269  *

   270  * See http://www.rfc-editor.org/rfc/rfc2047.txt for more information.

   271  *

   272  * Notes:

   273  * - Only encode strings that contain non-ASCII characters.

   274  * - We progressively cut-off a chunk with truncate_utf8(). This is to ensure

   275  *   each chunk starts and ends on a character boundary.

   276  * - Using \n as the chunk separator may cause problems on some systems and may

   277  *   have to be changed to \r\n or \r.

   278  */

   279 function mime_header_encode($string) {

   280   if (preg_match('/[^\x20-\x7E]/', $string)) {

   281     $chunk_size = 47; // floor((75 - strlen("=?UTF-8?B??=")) * 0.75);

   282     $len = strlen($string);

   283     $output = '';

   284     while ($len > 0) {

   285       $chunk = drupal_truncate_bytes($string, $chunk_size);

   286       $output .= ' =?UTF-8?B?'. base64_encode($chunk) ."?=\n";

   287       $c = strlen($chunk);

   288       $string = substr($string, $c);

   289       $len -= $c;

   290     }

   291     return trim($output);

   292   }

   293   return $string;

   294 }

   295 

   296 /**

   297  * Complement to mime_header_encode

   298  */

   299 function mime_header_decode($header) {

   300   // First step: encoded chunks followed by other encoded chunks (need to collapse whitespace)

   301   $header = preg_replace_callback('/=\?([^?]+)\?(Q|B)\?([^?]+|\?(?!=))\?=\s+(?==\?)/', '_mime_header_decode', $header);

   302   // Second step: remaining chunks (do not collapse whitespace)

   303   return preg_replace_callback('/=\?([^?]+)\?(Q|B)\?([^?]+|\?(?!=))\?=/', '_mime_header_decode', $header);

   304 }

   305 

   306 /**

   307  * Helper function to mime_header_decode

   308  */

   309 function _mime_header_decode($matches) {

   310   // Regexp groups:

   311   // 1: Character set name

   312   // 2: Escaping method (Q or B)

   313   // 3: Encoded data

   314   $data = ($matches[2] == 'B') ? base64_decode($matches[3]) : str_replace('_', ' ', quoted_printable_decode($matches[3]));

   315   if (strtolower($matches[1]) != 'utf-8') {

   316     $data = drupal_convert_to_utf8($data, $matches[1]);

   317   }

   318   return $data;

   319 }

   320 

   321 /**

   322  * Decode all HTML entities (including numerical ones) to regular UTF-8 bytes.

   323  * Double-escaped entities will only be decoded once ("&amp;lt;" becomes "&lt;", not "<").

   324  *

   325  * @param $text

   326  *   The text to decode entities in.

   327  * @param $exclude

   328  *   An array of characters which should not be decoded. For example,

   329  *   array('<', '&', '"'). This affects both named and numerical entities.

   330  */

   331 function decode_entities($text, $exclude = array()) {

   332   static $table;

   333   // We store named entities in a table for quick processing.

   334   if (!isset($table)) {

   335     // Get all named HTML entities.

   336     $table = array_flip(get_html_translation_table(HTML_ENTITIES));

   337     // PHP gives us ISO-8859-1 data, we need UTF-8.

   338     $table = array_map('utf8_encode', $table);

   339     // Add apostrophe (XML)

   340     $table['''] = "'";

   341   }

   342   $newtable = array_diff($table, $exclude);

   343 

   344   // Use a regexp to select all entities in one pass, to avoid decoding double-escaped entities twice.

   345   return preg_replace('/&(#x?)?([A-Za-z0-9]+);/e', '_decode_entities("$1", "$2", "$0", $newtable, $exclude)', $text);

   346 }

   347 

   348 /**

   349  * Helper function for decode_entities

   350  */

   351 function _decode_entities($prefix, $codepoint, $original, &$table, &$exclude) {

   352   // Named entity

   353   if (!$prefix) {

   354     if (isset($table[$original])) {

   355       return $table[$original];

   356     }

   357     else {

   358       return $original;

   359     }

   360   }

   361   // Hexadecimal numerical entity

   362   if ($prefix == '#x') {

   363     $codepoint = base_convert($codepoint, 16, 10);

   364   }

   365   // Decimal numerical entity (strip leading zeros to avoid PHP octal notation)

   366   else {

   367     $codepoint = preg_replace('/^0+/', '', $codepoint);

   368   }

   369   // Encode codepoint as UTF-8 bytes

   370   if ($codepoint < 0x80) {

   371     $str = chr($codepoint);

   372   }

   373   else if ($codepoint < 0x800) {

   374     $str = chr(0xC0 | ($codepoint >> 6))

   375          . chr(0x80 | ($codepoint & 0x3F));

   376   }

   377   else if ($codepoint < 0x10000) {

   378     $str = chr(0xE0 | ( $codepoint >> 12))

   379          . chr(0x80 | (($codepoint >> 6) & 0x3F))

   380          . chr(0x80 | ( $codepoint       & 0x3F));

   381   }

   382   else if ($codepoint < 0x200000) {

   383     $str = chr(0xF0 | ( $codepoint >> 18))

   384          . chr(0x80 | (($codepoint >> 12) & 0x3F))

   385          . chr(0x80 | (($codepoint >> 6)  & 0x3F))

   386          . chr(0x80 | ( $codepoint        & 0x3F));

   387   }

   388   // Check for excluded characters

   389   if (in_array($str, $exclude)) {

   390     return $original;

   391   }

   392   else {

   393     return $str;

   394   }

   395 }

   396 

   397 /**

   398  * Count the amount of characters in a UTF-8 string. This is less than or

   399  * equal to the byte count.

   400  */

   401 function drupal_strlen($text) {

   402   global $multibyte;

   403   if ($multibyte == UNICODE_MULTIBYTE) {

   404     return mb_strlen($text);

   405   }

   406   else {

   407     // Do not count UTF-8 continuation bytes.

   408     return strlen(preg_replace("/[\x80-\xBF]/", '', $text));

   409   }

   410 }

   411 

   412 /**

   413  * Uppercase a UTF-8 string.

   414  */

   415 function drupal_strtoupper($text) {

   416   global $multibyte;

   417   if ($multibyte == UNICODE_MULTIBYTE) {

   418     return mb_strtoupper($text);

   419   }

   420   else {

   421     // Use C-locale for ASCII-only uppercase

   422     $text = strtoupper($text);

   423     // Case flip Latin-1 accented letters

   424     $text = preg_replace_callback('/\xC3[\xA0-\xB6\xB8-\xBE]/', '_unicode_caseflip', $text);

   425     return $text;

   426   }

   427 }

   428 

   429 /**

   430  * Lowercase a UTF-8 string.

   431  */

   432 function drupal_strtolower($text) {

   433   global $multibyte;

   434   if ($multibyte == UNICODE_MULTIBYTE) {

   435     return mb_strtolower($text);

   436   }

   437   else {

   438     // Use C-locale for ASCII-only lowercase

   439     $text = strtolower($text);

   440     // Case flip Latin-1 accented letters

   441     $text = preg_replace_callback('/\xC3[\x80-\x96\x98-\x9E]/', '_unicode_caseflip', $text);

   442     return $text;

   443   }

   444 }

   445 

   446 /**

   447  * Helper function for case conversion of Latin-1.

   448  * Used for flipping U+C0-U+DE to U+E0-U+FD and back.

   449  */

   450 function _unicode_caseflip($matches) {

   451   return $matches[0][0] . chr(ord($matches[0][1]) ^ 32);

   452 }

   453 

   454 /**

   455  * Capitalize the first letter of a UTF-8 string.

   456  */

   457 function drupal_ucfirst($text) {

   458   // Note: no mbstring equivalent!

   459   return drupal_strtoupper(drupal_substr($text, 0, 1)) . drupal_substr($text, 1);

   460 }

   461 

   462 /**

   463  * Cut off a piece of a string based on character indices and counts. Follows

   464  * the same behavior as PHP's own substr() function.

   465  *

   466  * Note that for cutting off a string at a known character/substring

   467  * location, the usage of PHP's normal strpos/substr is safe and

   468  * much faster.

   469  */

   470 function drupal_substr($text, $start, $length = NULL) {

   471   global $multibyte;

   472   if ($multibyte == UNICODE_MULTIBYTE) {

   473     return $length === NULL ? mb_substr($text, $start) : mb_substr($text, $start, $length);

   474   }

   475   else {

   476     $strlen = strlen($text);

   477     // Find the starting byte offset

   478     $bytes = 0;

   479     if ($start > 0) {

   480       // Count all the continuation bytes from the start until we have found

   481       // $start characters

   482       $bytes = -1; $chars = -1;

   483       while ($bytes < $strlen && $chars < $start) {

   484         $bytes++;

   485         $c = ord($text[$bytes]);

   486         if ($c < 0x80 || $c >= 0xC0) {

   487           $chars++;

   488         }

   489       }

   490     }

   491     else if ($start < 0) {

   492       // Count all the continuation bytes from the end until we have found

   493       // abs($start) characters

   494       $start = abs($start);

   495       $bytes = $strlen; $chars = 0;

   496       while ($bytes > 0 && $chars < $start) {

   497         $bytes--;

   498         $c = ord($text[$bytes]);

   499         if ($c < 0x80 || $c >= 0xC0) {

   500           $chars++;

   501         }

   502       }

   503     }

   504     $istart = $bytes;

   505 

   506     // Find the ending byte offset

   507     if ($length === NULL) {

   508       $bytes = $strlen - 1;

   509     }

   510     else if ($length > 0) {

   511       // Count all the continuation bytes from the starting index until we have

   512       // found $length + 1 characters. Then backtrack one byte.

   513       $bytes = $istart; $chars = 0;

   514       while ($bytes < $strlen && $chars < $length) {

   515         $bytes++;

   516         $c = ord($text[$bytes]);

   517         if ($c < 0x80 || $c >= 0xC0) {

   518           $chars++;

   519         }

   520       }

   521       $bytes--;

   522     }

   523     else if ($length < 0) {

   524       // Count all the continuation bytes from the end until we have found

   525       // abs($length) characters

   526       $length = abs($length);

   527       $bytes = $strlen - 1; $chars = 0;

   528       while ($bytes >= 0 && $chars < $length) {

   529         $c = ord($text[$bytes]);

   530         if ($c < 0x80 || $c >= 0xC0) {

   531           $chars++;

   532         }

   533         $bytes--;

   534       }

   535     }

   536     $iend = $bytes;

   537 

   538     return substr($text, $istart, max(0, $iend - $istart + 1));

   539   }

   540 }

   541 

   542