All formats

uchardet

LanguageC++ with a C API
LicenseMPL-1.1, or GPL-2.0-or-later, or LGPL-2.1-or-later (your choice)
Repositorygitlab.freedesktop.org/uchardet/uchardet
Documentationuchardet home page, comments in uchardet.h
Depends onnothing
Read / writedetects the encoding; doesn't convert

uchardet is the encoding detector from Mozilla’s browser, extracted into a standalone library and extended with more languages and encodings. It recognizes UTF-8, UTF-16/32, the ISO-8859 and Windows code pages, KOI8-R, IBM866, Shift_JIS, EUC-JP, ISO-2022-JP, GB18030, Big5, EUC-KR and others, and also guesses the language. It’s used by mpv and other programs, and ships a uchardet command line tool.

It returns encoding names in the iconv style (WINDOWS-1252, UTF-8, SHIFT_JIS), so you can pass the result straight to iconv_open. It returns an empty string when it can’t decide, which happens for pure ASCII and for short texts.

Installation#

Shell
# Debian / Ubuntu
sudo apt install libuchardet-dev
# Fedora
sudo dnf install uchardet-devel
# macOS
brew install uchardet
# Windows (vcpkg)
vcpkg install uchardet

With pkg-config: cc prog.c $(pkg-config --cflags --libs uchardet).

Basic usage: detect the encoding of a file#

C
#include <stdio.h>
#include <uchardet.h>

int main(int argc, char **argv) {
    if (argc < 2) return 1;
    FILE *f = fopen(argv[1], "rb");
    if (!f) { perror(argv[1]); return 1; }

    uchardet_t ud = uchardet_new();
    char buf[65536];
    size_t n;
    while ((n = fread(buf, 1, sizeof(buf), f)) > 0) {
        if (uchardet_handle_data(ud, buf, n) != 0) {
            fprintf(stderr, "out of memory\n");
            break;
        }
    }
    fclose(f);
    uchardet_data_end(ud);

    const char *charset = uchardet_get_charset(ud);
    printf("%s\n", *charset ? charset : "unknown (ASCII or too little text)");
    uchardet_delete(ud);
    return 0;
}

How to detect the encoding and convert to UTF-8#

Pass the detected name to iconv:

C
#include <iconv.h>
#include <string.h>
#include <uchardet.h>

/* returns 0 on success; out receives UTF-8 */
int to_utf8(const char *in, size_t len, char *out, size_t out_cap) {
    uchardet_t ud = uchardet_new();
    uchardet_handle_data(ud, in, len);
    uchardet_data_end(ud);
    const char *cs = uchardet_get_charset(ud);
    const char *from = *cs ? cs : "WINDOWS-1252";   /* fallback */

    iconv_t cd = iconv_open("UTF-8", from);
    uchardet_delete(ud);   /* 'cs' is invalid after this, iconv_open copied it */
    if (cd == (iconv_t)-1) return -1;

    char *src = (char *)in, *dst = out;
    size_t src_left = len, dst_left = out_cap - 1;
    size_t r = iconv(cd, &src, &src_left, &dst, &dst_left);
    iconv_close(cd);
    if (r == (size_t)-1) return -1;
    *dst = '\0';
    return 0;
}

How to detect the encoding of a string in memory#

C
const char *text = "Gr\xFC\xDF Gott, sch\xF6ne Gr\xFC\xDF" "e aus M\xFCnchen";
uchardet_t ud = uchardet_new();
uchardet_handle_data(ud, text, strlen(text));
uchardet_data_end(ud);
printf("%s\n", uchardet_get_charset(ud));   /* likely ISO-8859-1 or WINDOWS-1252 */
uchardet_reset(ud);                         /* reuse for the next text */
uchardet_delete(ud);

How to detect from the command line#

Shell
uchardet notes.txt
# WINDOWS-1252

Tips#