All formats

ICU (International Components for Unicode)

LanguageC and C++ (ICU4C)
LicenseUnicode License v3 (permissive)
Repositorygithub.com/unicode-org/icu
DocumentationICU User Guide: Conversion, Character set detection, API reference
Depends onnothing (its data library is ~30 MB, can be trimmed)
Read / writedetects encodings; converts to and from 200+ encodings

ICU is the reference implementation of Unicode algorithms, used by Chrome, Android, Java, Node.js, PostgreSQL and many others. For text files two parts matter:

ICU is big, but it’s already on most systems: Linux distributions, macOS (as libicucore, without public headers), and Windows 10 1903+ (icu.dll, which you can link with icu.lib from the Windows SDK). Detection covers about 30 encodings (UTF-8/16/32, ISO-8859 and Windows code pages, KOI8-R, Shift_JIS, EUC-JP, ISO-2022, GB18030, Big5, EUC-KR, some EBCDIC), fewer than uchardet.

Installation#

Shell
# Debian / Ubuntu
sudo apt install libicu-dev
# Fedora
sudo dnf install libicu-devel
# macOS (keg-only: pass -I and -L from brew --prefix icu4c)
brew install icu4c
# Windows (vcpkg)
vcpkg install icu

Link with -licui18n -licuuc -licudata (detection is in icui18n, converters in icuuc), or pkg-config --libs icu-i18n icu-uc. With CMake: find_package(ICU REQUIRED COMPONENTS i18n uc) and link ICU::i18n ICU::uc.

Basic usage: detect the encoding and convert to UTF-8#

C
#include <stdio.h>
#include <stdlib.h>
#include <unicode/ucnv.h>
#include <unicode/ucsdet.h>

int main(int argc, char **argv) {
    if (argc < 2) return 1;
    FILE *f = fopen(argv[1], "rb");
    if (!f) return 1;
    fseek(f, 0, SEEK_END);
    long len = ftell(f);
    fseek(f, 0, SEEK_SET);
    char *data = malloc(len);
    fread(data, 1, len, f);
    fclose(f);

    UErrorCode status = U_ZERO_ERROR;
    UCharsetDetector *csd = ucsdet_open(&status);
    ucsdet_setText(csd, data, (int32_t)len, &status);
    const UCharsetMatch *match = ucsdet_detect(csd, &status);
    if (U_FAILURE(status) || !match) {
        fprintf(stderr, "detection failed: %s\n", u_errorName(status));
        return 1;
    }
    const char *name = ucsdet_getName(match, &status);
    int32_t confidence = ucsdet_getConfidence(match, &status);
    fprintf(stderr, "%s (confidence %d)\n", name, confidence);

    /* convert from the detected encoding to UTF-8 in one call */
    int32_t cap = (int32_t)len * 4 + 1;
    char *utf8 = malloc(cap);
    int32_t n = ucnv_convert("UTF-8", name, utf8, cap, data, (int32_t)len, &status);
    if (U_SUCCESS(status)) fwrite(utf8, 1, n, stdout);

    ucsdet_close(csd);   /* 'name' is owned by the detector */
    free(utf8);
    free(data);
    return U_SUCCESS(status) ? 0 : 1;
}

How to list all candidate encodings#

C
int32_t count = 0;
const UCharsetMatch **matches = ucsdet_detectAll(csd, &count, &status);
for (int32_t i = 0; i < count && U_SUCCESS(status); i++) {
    printf("%-12s %3d  %s\n",
           ucsdet_getName(matches[i], &status),
           ucsdet_getConfidence(matches[i], &status),
           ucsdet_getLanguage(matches[i], &status));
}

How to convert Windows-1252 to UTF-16 (ICU’s internal format)#

C
#include <unicode/ucnv.h>
#include <unicode/ustring.h>

UErrorCode status = U_ZERO_ERROR;
UConverter *conv = ucnv_open("windows-1252", &status);
UChar ubuf[1024];
int32_t ulen = ucnv_toUChars(conv, ubuf, 1024, "caf\xE9", 4, &status);
ucnv_close(conv);
/* ubuf now holds u"café", ulen == 4 */

How to fail on invalid bytes instead of substituting#

By default converters replace invalid input with U+FFFD (or ? / 0x1A when encoding). Install the “stop” callback to get an error instead:

C
UConverter *conv = ucnv_open("UTF-8", &status);
ucnv_setToUCallBack(conv, UCNV_TO_U_CALLBACK_STOP, NULL, NULL, NULL, &status);
int32_t ulen = ucnv_toUChars(conv, ubuf, 1024, bytes, nbytes, &status);
if (status == U_ILLEGAL_CHAR_FOUND || status == U_INVALID_CHAR_FOUND) {
    /* not valid UTF-8 */
}
ucnv_close(conv);

Tips#