All formats

iconv (GNU libiconv)

LanguageC (POSIX API)
Licenseglibc: LGPL-2.1; GNU libiconv: LGPL-2.1 (library), GPL-3.0 (the iconv program)
RepositoryGNU libiconv (savannah.gnu.org); glibc's version is part of glibc
DocumentationPOSIX iconv(), glibc manual: Generic Charset Conversion
Depends onnothing
Read / writeconverts between encodings; no detection

iconv_open, iconv and iconv_close are the standard Unix API for character set conversion. On Linux it’s part of glibc; on macOS and the BSDs it’s in the C library (macOS uses GNU libiconv). On Windows and other systems, GNU libiconv provides the same API. Both support all common encodings: UTF-8/16/32, ISO-8859-x, Windows code pages, KOI8, Shift_JIS, EUC-JP, GBK, GB18030, Big5, EUC-KR and more.

iconv converts streams in chunks and reports exactly where invalid or incomplete input is, which makes it good for large files. It doesn’t detect encodings: combine it with uchardet. Encoding names and extensions (//TRANSLIT, //IGNORE) differ a little between implementations.

Installation#

Nothing to install on Linux and macOS: include <iconv.h>. On macOS link with -liconv. Elsewhere:

Shell
# Windows (vcpkg)
vcpkg install libiconv
# MSYS2
pacman -S mingw-w64-ucrt-x86_64-libiconv

With CMake: find_package(Iconv REQUIRED) and link Iconv::Iconv (works with both glibc and libiconv).

Basic usage: convert a file from Windows-1252 to UTF-8#

C
#include <errno.h>
#include <iconv.h>
#include <stdio.h>

int main(int argc, char **argv) {
    if (argc < 3) return 1;
    FILE *in = fopen(argv[1], "rb"), *out = fopen(argv[2], "wb");
    if (!in || !out) return 1;

    iconv_t cd = iconv_open("UTF-8", "WINDOWS-1252");   /* to, from */
    if (cd == (iconv_t)-1) { perror("iconv_open"); return 1; }

    char inbuf[4096], outbuf[16384];
    size_t carry = 0, n;
    while ((n = fread(inbuf + carry, 1, sizeof(inbuf) - carry, in)) > 0 || carry) {
        char *src = inbuf, *dst = outbuf;
        size_t src_left = carry + n, dst_left = sizeof(outbuf);
        size_t r = iconv(cd, &src, &src_left, &dst, &dst_left);
        fwrite(outbuf, 1, dst - outbuf, out);
        if (r == (size_t)-1 && errno == EILSEQ) {
            fprintf(stderr, "invalid byte at input offset %ld\n", ftell(in) - (long)src_left);
            break;
        }
        /* EINVAL: incomplete multibyte sequence at the end, keep it for the next read */
        for (size_t i = 0; i < src_left; i++) inbuf[i] = src[i];
        carry = src_left;
        if (n == 0) break;   /* end of file with leftover bytes */
    }
    iconv_close(cd);
    fclose(in);
    fclose(out);
    return 0;
}

How to convert a string in memory#

C
#include <iconv.h>
#include <string.h>

/* returns the number of bytes written to out, or (size_t)-1 */
size_t convert(const char *to, const char *from, const char *in, size_t len, char *out, size_t cap) {
    iconv_t cd = iconv_open(to, from);
    if (cd == (iconv_t)-1) return (size_t)-1;
    char *src = (char *)in, *dst = out;
    size_t src_left = len, dst_left = cap;
    size_t r = iconv(cd, &src, &src_left, &dst, &dst_left);
    iconv(cd, NULL, NULL, &dst, &dst_left);   /* flush shift state (ISO-2022-JP etc.) */
    iconv_close(cd);
    return r == (size_t)-1 ? (size_t)-1 : (size_t)(dst - out);
}

/* convert("UTF-16LE", "UTF-8", "héllo", 6, buf, sizeof(buf)) */

How to replace characters that don’t exist in the target encoding#

Append //TRANSLIT to the target to approximate (é becomes e or ? in ASCII), or //IGNORE to drop them. Both are GNU extensions, supported by glibc and libiconv:

C
iconv_t cd = iconv_open("ASCII//TRANSLIT", "UTF-8");

How to convert from the command line#

Shell
iconv -f WINDOWS-1252 -t UTF-8 old.txt > new.txt
iconv -f UTF-8 -t ASCII//TRANSLIT names.txt
iconv -l    # list supported encodings

Tips#