Menu

- Learn
- Documentation
  - Language Reference
  - Library Reference
  - Command-line Reference
  - Feature Overview
  - Articles
- Downloads
- Packages
- Community
  - Blog
  - Orgs using D
  - Twitter
  - Forums
  - IRC
  - Community Discord
  - Wiki
  - GitHub
  - Issues
  - Get involved
  - Contributors
  - Foundation
  - Security Team
  - Donate
  - Sponsors
- Resources
  - Tour
  - Books
  - Tutorials
  - Tools
  - Editors
  - IDEs
  - run.dlang.io
  - Visual D
  - Acknowledgments
  - D Style
  - Glossary
  - Sitemap

Search
Report a bug
If you spot a problem with this page, click here to create a Bugzilla issue.
Improve this page
Quickly fork, edit online, and submit a pull request for this page. Requires a signed-in GitHub account. This works well for small changes. If you'd like to make larger changes you may want to consider using a local clone.

# std.utf

Encode and decode UTF-8, UTF-16 and UTF-32 strings.
UTF character support is restricted to '\u0000' <= character <= '\U0010FFFF'.

Category | Functions
--- | ---
Decode | decode decodeFront
Lazy decode | byCodeUnit byChar byWchar byDchar byUTF
Encode | encode toUTF8 toUTF16 toUTF32 toUTFz toUTF16z
Length | codeLength count stride strideBack
Index | toUCSindex toUTFindex
Validation | isValidDchar isValidCodepoint validate
Miscellaneous | replacementDchar UseReplacementDchar UTFException

See Also:
Wikipedia
http://www.cl.cam.ac.uk/~mgk25/unicode.html#utf-8
http://anubis.dkuug.dk/JTC1/SC2/WG2/docs/n1335
License:
Boost License 1.0.
Authors:
Walter Bright and Jonathan M Davis

Source std/utf.d

class `UTFException`: core.exception.UnicodeException;
  Exception thrown on errors in std.utf functions.
  Examples:

      import std.exception : assertThrown;

      char[4] buf;
      assertThrown!UTFException(encode(buf, cast(dchar) 0xD800));
      assertThrown!UTFException(encode(buf, cast(dchar) 0xDBFF));
      assertThrown!UTFException(encode(buf, cast(dchar) 0xDC00));
      assertThrown!UTFException(encode(buf, cast(dchar) 0xDFFF));
      assertThrown!UTFException(encode(buf, cast(dchar) 0x110000));

  pure nothrow @nogc @safe this(string `msg`, string `file` = __FILE__, size_t `line` = __LINE__, Throwable `next` = null);
  pure nothrow @safe this(string `msg`, size_t `index`, string `file` = __FILE__, size_t `line` = __LINE__, Throwable `next` = null);
    Standard exception constructors.
  const string `toString`();
    Returns:
    A string detailing the invalid UTF sequence.
pure nothrow @nogc @safe bool `isValidDchar`(dchar `c`);
  Check whether the given Unicode code point is valid.
  Parameters:

  dchar `c` | code point to check

  Returns:
  true if and only if `c` is a valid Unicode code point

  Note '\uFFFE' and '\uFFFF' are considered valid by `isValidDchar`, as they are permitted for internal use by an application, but they are not allowed for interchange by the Unicode standard.

  Examples:

      assert( isValidDchar(cast(dchar) 0x41));
      assert( isValidDchar(cast(dchar) 0x00));
      assert(!isValidDchar(cast(dchar) 0xD800));
      assert(!isValidDchar(cast(dchar) 0x11FFFF));

bool `isValidCodepoint`(Char)(Char `c`)
if (isSomeChar!Char);
  Checks if a single character forms a valid code point.
  When standing alone, some characters are invalid code points. For example the wchar 0xD800 is a so called high surrogate, which can only be interpreted together with a low surrogate following it. As a standalone character it is considered invalid.
  See Unicode Standard, D90, D91 and D92 for more details.
  Parameters:

  Char `c` | character to test
  Char | character type of `c`

  Returns:
  true, if `c` forms a valid code point.
  Examples:

      assert( isValidCodepoint(cast(char) 0x40));
      assert(!isValidCodepoint(cast(char) 0x80));
      assert( isValidCodepoint(cast(wchar) 0x1234));
      assert(!isValidCodepoint(cast(wchar) 0xD800));
      assert( isValidCodepoint(cast(dchar) 0x0010FFFF));
      assert(!isValidCodepoint(cast(dchar) 0x12345678));

uint `stride`(S)(auto ref S `str`, size_t `index`)
if (is(S : const(char[])) || isRandomAccessRange!S && is(immutable(ElementType!S) == immutable(char)));
uint `stride`(S)(auto ref S `str`)
if (is(S : const(char[])) || isInputRange!S && is(immutable(ElementType!S) == immutable(char)));
uint `stride`(S)(auto ref S `str`, size_t `index`)
if (is(S : const(wchar[])) || isRandomAccessRange!S && is(immutable(ElementType!S) == immutable(wchar)));
pure @safe uint `stride`(S)(auto ref S `str`)
if (is(S : const(wchar[])));
uint `stride`(S)(auto ref S `str`)
if (isInputRange!S && is(immutable(ElementType!S) == immutable(wchar)) && !is(S : const(wchar[])));
uint `stride`(S)(auto ref S `str`, size_t `index` = 0)
if (is(S : const(dchar[])) || isInputRange!S && is(immutable(ElementEncodingType!S) == immutable(dchar)));
  Calculate the length of the UTF sequence starting at `index` in `str`.
  Parameters:

  S `str` | input range of UTF code units. Must be random access if `index` is passed
  size_t `index` | starting index of UTF sequence (default: 0)

  Returns:
  The number of code units in the UTF sequence. For UTF-8, this is a value between 1 and 4 (as per RFC 3629, section 3). For UTF-16, it is either 1 or 2. For UTF-32, it is always 1.
  Throws:
  May throw a UTFException if `str`[`index`] is not the start of a valid UTF sequence.

  Note `stride` will only analyze the first `str`[`index`] element. It will not fully verify the validity of the UTF sequence, nor even verify the presence of the sequence: it will not actually guarantee that index + stride(str, index) <= str.length.

  Examples:

      writeln("a".stride); // 1
      writeln("λ".stride); // 2
      writeln("aλ".stride); // 1
      writeln("aλ".stride(1)); // 2
      writeln("𐐷".stride); // 4

uint `strideBack`(S)(auto ref S `str`, size_t `index`)
if (is(S : const(char[])) || isRandomAccessRange!S && is(immutable(ElementType!S) == immutable(char)));
uint `strideBack`(S)(auto ref S `str`)
if (is(S : const(char[])) || isRandomAccessRange!S && hasLength!S && is(immutable(ElementType!S) == immutable(char)));
uint `strideBack`(S)(auto ref S `str`)
if (isBidirectionalRange!S && is(immutable(ElementType!S) == immutable(char)) && !isRandomAccessRange!S);
uint `strideBack`(S)(auto ref S `str`, size_t `index`)
if (is(S : const(wchar[])) || isRandomAccessRange!S && is(immutable(ElementType!S) == immutable(wchar)));
uint `strideBack`(S)(auto ref S `str`)
if (is(S : const(wchar[])) || isBidirectionalRange!S && is(immutable(ElementType!S) == immutable(wchar)));
uint `strideBack`(S)(auto ref S `str`, size_t `index`)
if (isRandomAccessRange!S && is(immutable(ElementEncodingType!S) == immutable(dchar)));
uint `strideBack`(S)(auto ref S `str`)
if (isBidirectionalRange!S && is(immutable(ElementEncodingType!S) == immutable(dchar)));
  Calculate the length of the UTF sequence ending one code unit before `index` in `str`.
  Parameters:

  S `str` | bidirectional range of UTF code units. Must be random access if `index` is passed
  size_t `index` | index one past end of UTF sequence (default: `str`.length)

  Returns:
  The number of code units in the UTF sequence. For UTF-8, this is a value between 1 and 4 (as per RFC 3629, section 3). For UTF-16, it is either 1 or 2. For UTF-32, it is always 1.
  Throws:
  May throw a UTFException if `str`[`index`] is not one past the end of a valid UTF sequence.

  Note `strideBack` will only analyze the element at str[index - 1] element. It will not fully verify the validity of the UTF sequence, nor even verify the presence of the sequence: it will not actually guarantee that strideBack(str, index) <= index.

  Examples:

      writeln("a".strideBack); // 1
      writeln("λ".strideBack); // 2
      writeln("aλ".strideBack); // 2
      writeln("aλ".strideBack(1)); // 1
      writeln("𐐷".strideBack); // 4

pure @safe size_t `toUCSindex`(C)(const(C)[] `str`, size_t `index`)
if (isSomeChar!C);
  Given `index` into `str` and assuming that `index` is at the start of a UTF sequence, `toUCSindex` determines the number of UCS characters up to `index`. So, `index` is the index of a code unit at the beginning of a code point, and the return value is how many code points into the string that that code point is.
  Examples:

      writeln(toUCSindex(`hello world`, 7)); // 7
      writeln(toUCSindex(`hello world`w, 7)); // 7
      writeln(toUCSindex(`hello world`d, 7)); // 7

      writeln(toUCSindex(`Ma Chérie`, 7)); // 6
      writeln(toUCSindex(`Ma Chérie`w, 7)); // 7
      writeln(toUCSindex(`Ma Chérie`d, 7)); // 7

      writeln(toUCSindex(`さいごの果実 / ミツバチと科学者`, 9)); // 3
      writeln(toUCSindex(`さいごの果実 / ミツバチと科学者`w, 9)); // 9
      writeln(toUCSindex(`さいごの果実 / ミツバチと科学者`d, 9)); // 9

pure @safe size_t `toUTFindex`(C)(const(C)[] `str`, size_t `n`)
if (isSomeChar!C);
  Given a UCS index `n` into `str`, returns the UTF index. So, `n` is how many code points into the string the code point is, and the array index of the code unit is returned.
  Examples:

      writeln(toUTFindex(`hello world`, 7)); // 7
      writeln(toUTFindex(`hello world`w, 7)); // 7
      writeln(toUTFindex(`hello world`d, 7)); // 7

      writeln(toUTFindex(`Ma Chérie`, 6)); // 7
      writeln(toUTFindex(`Ma Chérie`w, 7)); // 7
      writeln(toUTFindex(`Ma Chérie`d, 7)); // 7

      writeln(toUTFindex(`さいごの果実 / ミツバチと科学者`, 3)); // 9
      writeln(toUTFindex(`さいごの果実 / ミツバチと科学者`w, 9)); // 9
      writeln(toUTFindex(`さいごの果実 / ミツバチと科学者`d, 9)); // 9

alias `UseReplacementDchar` = std.typecons.Flag!"useReplacementDchar".Flag;
  Whether or not to replace invalid UTF with replacementDchar
dchar `decode`(UseReplacementDchar useReplacementDchar = No.useReplacementDchar, S)(auto ref S `str`, ref size_t `index`)
if (!isSomeString!S && isRandomAccessRange!S && hasSlicing!S && hasLength!S && isSomeChar!(ElementType!S));
pure @trusted dchar `decode`(UseReplacementDchar useReplacementDchar = No.useReplacementDchar, S)(auto ref scope S `str`, ref size_t `index`)
if (isSomeString!S);
  Decodes and returns the code point starting at `str`[`index`]. `index` is advanced to one past the decoded code point. If the code point is not well-formed, then a UTFException is thrown and `index` remains unchanged.
  decode will only work with strings and random access ranges of code units with length and slicing, whereas decodeFront will work with any input range of code units.
  Parameters:

  useReplacementDchar | if invalid UTF, return replacementDchar rather than throwing
  S `str` | input string or indexable Range
  size_t `index` | starting index into s[]; incremented by number of code units processed

  Returns:
  decoded character
  Throws:
  UTFException if `str`[`index`] is not the start of a valid UTF sequence and useReplacementDchar is No.useReplacementDchar
  Examples:

      size_t i;

      assert("a".decode(i) == 'a' && i == 1);
      i = 0;
      assert("å".decode(i) == 'å' && i == 2);
      i = 1;
      assert("aå".decode(i) == 'å' && i == 3);
      i = 0;
      assert("å"w.decode(i) == 'å' && i == 1);

      // ë as a multi-code point grapheme
      i = 0;
      assert("e\u0308".decode(i) == 'e' && i == 1);
      // ë as a single code point grapheme
      i = 0;
      assert("ë".decode(i) == 'ë' && i == 2);
      i = 0;
      assert("ë"w.decode(i) == 'ë' && i == 1);

dchar `decodeFront`(UseReplacementDchar useReplacementDchar = No.useReplacementDchar, S)(ref S `str`, out size_t `numCodeUnits`)
if (!isSomeString!S && isInputRange!S && isSomeChar!(ElementType!S));
pure @trusted dchar `decodeFront`(UseReplacementDchar useReplacementDchar = No.useReplacementDchar, S)(ref scope S `str`, out size_t `numCodeUnits`)
if (isSomeString!S);
dchar `decodeFront`(UseReplacementDchar useReplacementDchar = No.useReplacementDchar, S)(ref S `str`)
if (isInputRange!S && isSomeChar!(ElementType!S));
  `decodeFront` is a variant of decode which specifically decodes the first code point. Unlike decode, `decodeFront` accepts any input range of code units (rather than just a string or random access range). It also takes the range by ref and pops off the elements as it decodes them. If `numCodeUnits` is passed in, it gets set to the number of code units which were in the code point which was decoded.
  Parameters:

  useReplacementDchar | if invalid UTF, return replacementDchar rather than throwing
  S `str` | input string or indexable Range
  size_t `numCodeUnits` | set to number of code units processed

  Returns:
  decoded character
  Throws:
  UTFException if `str`.front is not the start of a valid UTF sequence. If an exception is thrown, then there is no guarantee as to the number of code units which were popped off, as it depends on the type of range being used and how many code units had to be popped off before the code point was determined to be invalid.
  Examples:

      import std.range.primitives;
      string str = "Hello, World!";

      assert(str.decodeFront == 'H' && str == "ello, World!");
      str = "å";
      assert(str.decodeFront == 'å' && str.empty);
      str = "å";
      size_t i;
      assert(str.decodeFront(i) == 'å' && i == 2 && str.empty);

dchar `decodeBack`(UseReplacementDchar useReplacementDchar = No.useReplacementDchar, S)(ref S `str`, out size_t `numCodeUnits`)
if (isSomeString!S);
dchar `decodeBack`(UseReplacementDchar useReplacementDchar = No.useReplacementDchar, S)(ref S `str`, out size_t `numCodeUnits`)
if (!isSomeString!S && isSomeChar!(ElementType!S) && isBidirectionalRange!S && (isRandomAccessRange!S && hasLength!S || !isRandomAccessRange!S));
dchar `decodeBack`(UseReplacementDchar useReplacementDchar = No.useReplacementDchar, S)(ref S `str`)
if (isSomeString!S || isRandomAccessRange!S && hasLength!S && isSomeChar!(ElementType!S) || !isRandomAccessRange!S && isBidirectionalRange!S && isSomeChar!(ElementType!S));
  `decodeBack` is a variant of decode which specifically decodes the last code point. Unlike decode, `decodeBack` accepts any bidirectional range of code units (rather than just a string or random access range). It also takes the range by ref and pops off the elements as it decodes them. If `numCodeUnits` is passed in, it gets set to the number of code units which were in the code point which was decoded.
  Parameters:

  useReplacementDchar | if invalid UTF, return replacementDchar rather than throwing
  S `str` | input string or bidirectional Range
  size_t `numCodeUnits` | gives the number of code units processed

  Returns:
  A decoded UTF character.
  Throws:
  UTFException if `str`.back is not the end of a valid UTF sequence. If an exception is thrown, the `str` itself remains unchanged, but there is no guarantee as to the value of `numCodeUnits` (when passed).
  Examples:

      import std.range.primitives;
      string str = "Hello, World!";

      assert(str.decodeBack == '!' && str == "Hello, World");
      str = "å";
      assert(str.decodeBack == 'å' && str.empty);
      str = "å";
      size_t i;
      assert(str.decodeBack(i) == 'å' && i == 2 && str.empty);

pure @safe size_t `encode`(UseReplacementDchar useReplacementDchar = No.useReplacementDchar)(out char[4] `buf`, dchar `c`);
pure @safe size_t `encode`(UseReplacementDchar useReplacementDchar = No.useReplacementDchar)(out wchar[2] `buf`, dchar `c`);
pure @safe size_t `encode`(UseReplacementDchar useReplacementDchar = No.useReplacementDchar)(out dchar[1] `buf`, dchar `c`);
  Encodes `c` into the static array, `buf`, and returns the actual length of the encoded character (a number between 1 and 4 for char[4] buffers and a number between 1 and 2 for wchar[2] buffers).
  Throws:
  UTFException if `c` is not a valid UTF code point.
  Examples:

      import std.exception : assertThrown;
      import std.typecons : Yes;

      char[4] buf;

      assert(encode(buf, '\u0000') == 1 && buf[0 .. 1] == "\u0000");
      assert(encode(buf, '\u007F') == 1 && buf[0 .. 1] == "\u007F");
      assert(encode(buf, '\u0080') == 2 && buf[0 .. 2] == "\u0080");
      assert(encode(buf, '\uE000') == 3 && buf[0 .. 3] == "\uE000");
      assert(encode(buf, 0xFFFE) == 3 && buf[0 .. 3] == "\xEF\xBF\xBE");
      assertThrown!UTFException(encode(buf, cast(dchar) 0x110000));

      encode!(Yes.useReplacementDchar)(buf, cast(dchar) 0x110000);
      auto slice = buf[];
      writeln(slice.decodeFront); // replacementDchar

  Examples:

      import std.exception : assertThrown;
      import std.typecons : Yes;

      wchar[2] buf;

      assert(encode(buf, '\u0000') == 1 && buf[0 .. 1] == "\u0000");
      assert(encode(buf, '\uD7FF') == 1 && buf[0 .. 1] == "\uD7FF");
      assert(encode(buf, '\uE000') == 1 && buf[0 .. 1] == "\uE000");
      assert(encode(buf, '\U00010000') == 2 && buf[0 .. 2] == "\U00010000");
      assert(encode(buf, '\U0010FFFF') == 2 && buf[0 .. 2] == "\U0010FFFF");
      assertThrown!UTFException(encode(buf, cast(dchar) 0xD800));

      encode!(Yes.useReplacementDchar)(buf, cast(dchar) 0x110000);
      auto slice = buf[];
      writeln(slice.decodeFront); // replacementDchar

  Examples:

      import std.exception : assertThrown;
      import std.typecons : Yes;

      dchar[1] buf;

      assert(encode(buf, '\u0000') == 1 && buf[0] == '\u0000');
      assert(encode(buf, '\uD7FF') == 1 && buf[0] == '\uD7FF');
      assert(encode(buf, '\uE000') == 1 && buf[0] == '\uE000');
      assert(encode(buf, '\U0010FFFF') == 1 && buf[0] == '\U0010FFFF');
      assertThrown!UTFException(encode(buf, cast(dchar) 0xD800));

      encode!(Yes.useReplacementDchar)(buf, cast(dchar) 0x110000);
      writeln(buf[0]); // replacementDchar

pure @safe void `encode`(UseReplacementDchar useReplacementDchar = No.useReplacementDchar)(ref scope char[] `str`, dchar `c`);
pure @safe void `encode`(UseReplacementDchar useReplacementDchar = No.useReplacementDchar)(ref scope wchar[] `str`, dchar `c`);
pure @safe void `encode`(UseReplacementDchar useReplacementDchar = No.useReplacementDchar)(ref scope dchar[] `str`, dchar `c`);
  Encodes `c` in `str`'s encoding and appends it to `str`.
  Throws:
  UTFException if `c` is not a valid UTF code point.
  Examples:

      char[] s = "abcd".dup;
      dchar d1 = 'a';
      dchar d2 = 'ø';

      encode(s, d1);
      writeln(s.length); // 5
      writeln(s); // "abcda"
      encode(s, d2);
      writeln(s.length); // 7
      writeln(s); // "abcdaø"

pure nothrow @nogc @safe ubyte `codeLength`(C)(dchar `c`)
if (isSomeChar!C);
  Returns the number of code units that are required to encode the code point `c` when C is the character type used to encode it.
  Examples:

      writeln(codeLength!char('a')); // 1
      writeln(codeLength!wchar('a')); // 1
      writeln(codeLength!dchar('a')); // 1

      writeln(codeLength!char('\U0010FFFF')); // 4
      writeln(codeLength!wchar('\U0010FFFF')); // 2
      writeln(codeLength!dchar('\U0010FFFF')); // 1

size_t `codeLength`(C, InputRange)(InputRange `input`)
if (isSomeFiniteCharInputRange!InputRange);
  Returns the number of code units that are required to encode str in a string whose character type is C. This is particularly useful when slicing one string with the length of another and the two string types use different character types.
  Parameters:

  C | the character type to get the encoding length for
  InputRange `input` | the input range to calculate the encoding length from

  Returns:
  The number of code units in `input` when encoded to C
  Examples:

      assert(codeLength!char("hello world") ==
             "hello world".length);
      assert(codeLength!wchar("hello world") ==
             "hello world"w.length);
      assert(codeLength!dchar("hello world") ==
             "hello world"d.length);

      assert(codeLength!char(`プログラミング`) ==
             `プログラミング`.length);
      assert(codeLength!wchar(`プログラミング`) ==
             `プログラミング`w.length);
      assert(codeLength!dchar(`プログラミング`) ==
             `プログラミング`d.length);

      string haystack = `Être sans la verité, ça, ce ne serait pas bien.`;
      wstring needle = `Être sans la verité`;
      assert(haystack[codeLength!char(needle) .. $] ==
             `, ça, ce ne serait pas bien.`);

pure @safe void `validate`(S)(in S `str`)
if (isSomeString!S);
  Checks to see if `str` is well-formed unicode or not.
  Throws:
  UTFException if `str` is not well-formed.
  Examples:

      import std.exception : assertThrown;
      char[] a = [167, 133, 175];
      assertThrown!UTFException(validate(a));

string `toUTF8`(S)(S `s`)
if (isSomeFiniteCharInputRange!S);
  Encodes the elements of `s` to UTF-8 and returns a newly allocated string of the elements.
  Parameters:

  S `s` | the string to encode

  Returns:
  A UTF-8 string
  See Also:
  For a lazy, non-allocating version of these functions, see byUTF.
  Examples:

      import std.algorithm.comparison : equal;

      // The ö is represented by two UTF-8 code units
      assert("Hellø"w.toUTF8.equal(['H', 'e', 'l', 'l', 0xC3, 0xB8]));

      // 𐐷 is four code units in UTF-8
      assert("𐐷"d.toUTF8.equal([0xF0, 0x90, 0x90, 0xB7]));

wstring `toUTF16`(S)(S `s`)
if (isSomeFiniteCharInputRange!S);
  Encodes the elements of `s` to UTF-16 and returns a newly GC allocated wstring of the elements.
  Parameters:

  S `s` | the range to encode

  Returns:
  A UTF-16 string
  See Also:
  For a lazy, non-allocating version of these functions, see byUTF.
  Examples:

      import std.algorithm.comparison : equal;

      // these graphemes are two code units in UTF-16 and one in UTF-32
      writeln("𤭢"d.length); // 1
      writeln("𐐷"d.length); // 1

      assert("𤭢"d.toUTF16.equal([0xD852, 0xDF62]));
      assert("𐐷"d.toUTF16.equal([0xD801, 0xDC37]));

dstring `toUTF32`(S)(scope S `s`)
if (isSomeFiniteCharInputRange!S);
  Encodes the elements of `s` to UTF-32 and returns a newly GC allocated dstring of the elements.
  Parameters:

  S `s` | the range to encode

  Returns:
  A UTF-32 string
  See Also:
  For a lazy, non-allocating version of these functions, see byUTF.
  Examples:

      import std.algorithm.comparison : equal;

      // these graphemes are two code units in UTF-16 and one in UTF-32
      writeln("𤭢"w.length); // 2
      writeln("𐐷"w.length); // 2

      assert("𤭢"w.toUTF32.equal([0x00024B62]));
      assert("𐐷"w.toUTF32.equal([0x00010437]));

template `toUTFz`(P) if (is(P == C*, C) && isSomeChar!C)
  Returns a C-style zero-terminated string equivalent to str. str must not contain embedded '\0''s as any C function will treat the first '\0' that it sees as the end of the string. If str.empty is true, then a string containing only '\0' is returned.
  `toUTFz` accepts any type of string and is templated on the type of character pointer that you wish to convert to. It will avoid allocating a new string if it can, but there's a decent chance that it will end up having to allocate a new string - particularly when dealing with character types other than char.
  Warning 1: If the result of `toUTFz` equals str.ptr, then if anything alters the character one past the end of str (which is the '\0' character terminating the string), then the string won't be zero-terminated anymore. The most likely scenarios for that are if you append to str and no reallocation takes place or when str is a slice of a larger array, and you alter the character in the larger array which is one character past the end of str. Another case where it could occur would be if you had a mutable character array immediately after str in memory (for example, if they're member variables in a user-defined type with one declared right after the other) and that character array happened to start with '\0'. Such scenarios will never occur if you immediately use the zero-terminated string after calling `toUTFz` and the C function using it doesn't keep a reference to it. Also, they are unlikely to occur even if you save the zero-terminated string (the cases above would be among the few examples of where it could happen). However, if you save the zero-terminate string and want to be absolutely certain that the string stays zero-terminated, then simply append a '\0' to the string and use its ptr property rather than calling `toUTFz`.
  Warning 2: When passing a character pointer to a C function, and the C function keeps it around for any reason, make sure that you keep a reference to it in your D code. Otherwise, it may go away during a garbage collection cycle and cause a nasty bug when the C code tries to use it.
  Examples:

      auto p1 = toUTFz!(char*)("hello world");
      auto p2 = toUTFz!(const(char)*)("hello world");
      auto p3 = toUTFz!(immutable(char)*)("hello world");
      auto p4 = toUTFz!(char*)("hello world"d);
      auto p5 = toUTFz!(const(wchar)*)("hello world");
      auto p6 = toUTFz!(immutable(dchar)*)("hello world"w);

pure @safe const(wchar)* `toUTF16z`(C)(const(C)[] `str`)
if (isSomeChar!C);
  `toUTF16z` is a convenience function for toUTFz!(const(wchar)*).
  Encodes string s into UTF-16 and returns the encoded string. `toUTF16z` is suitable for calling the 'W' functions in the Win32 API that take an LPCWSTR argument.
  Examples:

      string str = "Hello, World!";
      const(wchar)* p = str.toUTF16z;
      writeln(p[str.length]); // '\0'

pure nothrow @nogc @safe size_t `count`(C)(const(C)[] `str`)
if (isSomeChar!C);
  Returns the total number of code points encoded in `str`.

  Supercedes This function supercedes toUCSindex.

  Standards:
  Unicode 5.0, ASCII, ISO-8859-1, WINDOWS-1252
  Throws:
  UTFException if `str` is not well-formed.
  Examples:

      writeln(count("")); // 0
      writeln(count("a")); // 1
      writeln(count("abc")); // 3
      writeln(count("\u20AC100")); // 4

enum dchar `replacementDchar`;
  Inserted in place of invalid UTF sequences.

  References http://en.wikipedia.org/wiki/Replacement_character#Replacement_character

auto `byCodeUnit`(R)(R `r`)
if (isConvertibleToString!R && !isStaticArray!R || isInputRange!R && isSomeChar!(ElementEncodingType!R));
  Iterate a range of char, wchar, or dchars by code unit.
  The purpose is to bypass the special case decoding that std.range.primitives.front does to character arrays. As a result, using ranges with `byCodeUnit` can be nothrow while std.range.primitives.front throws when it encounters invalid Unicode sequences.
  A code unit is a building block of the UTF encodings. Generally, an individual code unit does not represent what's perceived as a full character (a.k.a. a grapheme cluster in Unicode terminology). Many characters are encoded with multiple code units. For example, the UTF-8 code units for ø are 0xC3 0xB8. That means, an individual element of `byCodeUnit` often does not form a character on its own. Attempting to treat it as one while iterating over the resulting range will give nonsensical results.
  Parameters:

  R `r` | an input range of characters (including strings) or a type that implicitly converts to a string type.

  Returns:
  If `r` is not an auto-decodable string (i.e. a narrow string or a user-defined type that implicitly converts to a string type), then `r` is returned.
  Otherwise, `r` is converted to its corresponding string type (if it's not already a string) and wrapped in a random-access range where the element encoding type of the string (its code unit) is the element type of the range, and that range returned. The range has slicing.
  If `r` is quirky enough to be a struct or class which is an input range of characters on its own (i.e. it has the input range API as member functions), and it's implicitly convertible to a string type, then `r` is returned, and no implicit conversion takes place.
  If `r` is wrapped in a new range, then that range has a source property for returning the string that's currently contained within that range.
  See Also:
  Refer to the std.uni docs for a reference on Unicode terminology.
  For a range that iterates by grapheme cluster (written character) see std.uni.byGrapheme.
  Examples:

      import std.range.primitives;
      import std.traits : isAutodecodableString;

      auto r = "Hello, World!".byCodeUnit();
      static assert(hasLength!(typeof(r)));
      static assert(hasSlicing!(typeof(r)));
      static assert(isRandomAccessRange!(typeof(r)));
      static assert(is(ElementType!(typeof(r)) == immutable char));

      // contrast with the range capabilities of standard strings (with or
      // without autodecoding enabled).
      auto s = "Hello, World!";
      static assert(isBidirectionalRange!(typeof(r)));
      static if (isAutodecodableString!(typeof(s)))
      {
          // with autodecoding enabled, strings are non-random-access ranges of
          // dchar.
          static assert(is(ElementType!(typeof(s)) == dchar));
          static assert(!isRandomAccessRange!(typeof(s)));
          static assert(!hasSlicing!(typeof(s)));
          static assert(!hasLength!(typeof(s)));
      }
      else
      {
          // without autodecoding, strings are normal arrays.
          static assert(is(ElementType!(typeof(s)) == immutable char));
          static assert(isRandomAccessRange!(typeof(s)));
          static assert(hasSlicing!(typeof(s)));
          static assert(hasLength!(typeof(s)));
      }

  Examples:
  `byCodeUnit` does no Unicode decoding

      string noel1 = "noe\u0308l"; // noël using e + combining diaeresis
      assert(noel1.byCodeUnit[2] != 'ë');
      writeln(noel1.byCodeUnit[2]); // 'e'

      string noel2 = "no\u00EBl"; // noël using a precomposed ë character
      // Because string is UTF-8, the code unit at index 2 is just
      // the first of a sequence that encodes 'ë'
      assert(noel2.byCodeUnit[2] != 'ë');

  Examples:
  `byCodeUnit` exposes a source property when wrapping narrow strings.

      import std.algorithm.comparison : equal;
      import std.range : popFrontN;
      import std.traits : isAutodecodableString;
      {
          auto range = byCodeUnit("hello world");
          range.popFrontN(3);
          assert(equal(range.save, "lo world"));
          static if (isAutodecodableString!string) // only enabled with autodecoding
          {
              string str = range.source;
              writeln(str); // "lo world"
          }
      }
      // source only exists if the range was wrapped
      {
          auto range = byCodeUnit("hello world"d);
          static assert(!__traits(compiles, range.source));
      }

alias `byChar` = byUTF!(char, Flag.yes).byUTF(R)(R r) if (isAutodecodableString!R && isInputRange!R && isSomeChar!(ElementEncodingType!R));
alias `byWchar` = byUTF!(wchar, Flag.yes).byUTF(R)(R r) if (isAutodecodableString!R && isInputRange!R && isSomeChar!(ElementEncodingType!R));
alias `byDchar` = byUTF!(dchar, Flag.yes).byUTF(R)(R r) if (isAutodecodableString!R && isInputRange!R && isSomeChar!(ElementEncodingType!R));
  Iterate an input range of characters by char, wchar, or dchar. These aliases simply forward to byUTF with the corresponding C argument.
  Parameters:

  R r | input range of characters, or array of characters

template `byUTF`(C, UseReplacementDchar useReplacementDchar = Yes.useReplacementDchar) if (isSomeChar!C)
  Iterate an input range of characters by char type C by encoding the elements of the range.
  UTF sequences that cannot be converted to the specified encoding are either replaced by U+FFFD per "5.22 Best Practice for U+FFFD Substitution" of the Unicode Standard 6.2 or result in a thrown UTFException. Hence byUTF is not symmetric. This algorithm is lazy, and does not allocate memory. @nogc, pure-ity, nothrow, and @safe-ty are inferred from the r parameter.
  Parameters:

  C | char, wchar, or dchar
  useReplacementDchar | UseReplacementDchar.yes means replace invalid UTF with replacementDchar, UseReplacementDchar.no means throw UTFException for invalid UTF

  Throws:
  UTFException if invalid UTF sequence and useReplacementDchar is set to UseReplacementDchar.no

  GC Does not use GC if useReplacementDchar is set to UseReplacementDchar.yes

  Returns:
  A bidirectional range if R is a bidirectional range and not auto-decodable, as defined by std.traits.isAutodecodableString.
  A forward range if R is a forward range and not auto-decodable.
  Or, if R is a range and it is auto-decodable and is(ElementEncodingType!typeof(r) == C), then the range is passed to byCodeUnit.
  Otherwise, an input range of characters.
  Examples:

      import std.algorithm.comparison : equal;

      // hellö as a range of `char`s, which are UTF-8
      assert("hell\u00F6".byUTF!char().equal(['h', 'e', 'l', 'l', 0xC3, 0xB6]));

      // `wchar`s are able to hold the ö in a single element (UTF-16 code unit)
      assert("hell\u00F6".byUTF!wchar().equal(['h', 'e', 'l', 'l', 'ö']));

      // 𐐷 is four code units in UTF-8, two in UTF-16, and one in UTF-32
      assert("𐐷".byUTF!char().equal([0xF0, 0x90, 0x90, 0xB7]));
      assert("𐐷".byUTF!wchar().equal([0xD801, 0xDC37]));
      assert("𐐷".byUTF!dchar().equal([0x00010437]));

  Examples:

      import std.algorithm.comparison : equal;
      import std.exception : assertThrown;

      assert("hello\xF0betty".byChar.byUTF!(dchar, UseReplacementDchar.yes).equal("hello\uFFFDetty"));
      assertThrown!UTFException("hello\xF0betty".byChar.byUTF!(dchar, UseReplacementDchar.no).equal("hello betty"));

  Examples:

      import std.range.primitives;
      wchar[] s = ['ă', 'î'];

      auto rc = s.byUTF!char;
      static assert(isBidirectionalRange!(typeof(rc)));
      writeln(rc.back); // 0xae
      rc.popBack;
      writeln(rc.back); // 0xc3
      rc.popBack;
      writeln(rc.back); // 0x83
      rc.popBack;
      writeln(rc.back); // 0xc4

      auto rw = s.byUTF!wchar;
      static assert(isBidirectionalRange!(typeof(rw)));
      writeln(rw.back); // 'î'
      rw.popBack;
      writeln(rw.back); // 'ă'

      auto rd = s.byUTF!dchar;
      static assert(isBidirectionalRange!(typeof(rd)));
      writeln(rd.back); // 'î'
      rd.popBack;
      writeln(rd.back); // 'ă'

Copyright © 1999-2023 by the D Language Foundation | Page generated by Ddoc on Mon May 1 15:50:47 2023