/* begin file src/simdutf.cpp */

#include "simdutf.h"

/* begin file src/implementation.cpp */

// We should not get warnings while including <x86intrin.h> yet we do

// under some versions of GCC.

// If the x86intrin.h header has uninitialized values that are problematic,

SIMDUTF_DISABLE_GCC_WARNING(-Wuninitialized)

#endif

// We should not get warnings while including <x86intrin.h> yet we do

// under some versions of GCC.

// If the x86intrin.h header has uninitialized values that are problematic,

SIMDUTF_DISABLE_GCC_WARNING(-Wuninitialized)

#endif

// We should not get warnings while including <x86intrin.h> yet we do

// under some versions of GCC.

// If the x86intrin.h header has uninitialized values that are problematic,

SIMDUTF_DISABLE_GCC_WARNING(-Wuninitialized)

#endif

*

* The caller is responsible to ensure that len > 0.

*

* will be SIZE_T - 1, SIZE_T - 2, or SIZE_T - 3.

*/

template <endianness endian>

*

* The caller is responsible to ensure that len > 0.

*

* will be SIZE_T - 1, SIZE_T - 2, or SIZE_T - 3.

*/

inline result rewind_and_convert_with_errors(size_t prior_bytes, const char* buf, size_t len, char32_t* utf32_output) {

// 8 bytes when calling convert_masked_utf8_to_utf32. If you skip the last 16 bytes,

// and if the data is valid, then it is entirely safe because 16 UTF-8 bytes generate

// much more than 8 bytes. However, you cannot generally assume that you have valid

// to give us a good margin.

size_t leading_byte = 0;

size_t margin = size;

leading_byte += (int8_t(in[margin-1]) > -65);

}

// If the input is long enough, then we have that margin-1 is the fourth last leading byte.

// 8 bytes when calling convert_masked_utf8_to_utf32. If you skip the last 16 bytes,

// and if the data is valid, then it is entirely safe because 16 UTF-8 bytes generate

// much more than 8 bytes. However, you cannot generally assume that you have valid

// to give us a good margin.

size_t leading_byte = 0;

size_t margin = size;

leading_byte += (int8_t(in[margin-1]) > -65);

}

// If the input is long enough, then we have that margin-1 is the fourth last leading byte.

__m512i indexofthirdlastbytes = _mm512_add_epi16(mask_ffffffff,

indexofsecondlastbytes); // indices of the second last bytes

__m512i thirdlastbyte = _mm512_maskz_mov_epi8(m34,

__m512i thirdlastbytes = _mm512_maskz_permutexvar_epi8(0x5555555555555555, indexofthirdlastbytes,

thirdlastbyte); // the third last bytes (of three byte sequences, hi

// surrogate)

__m512i indexofthirdlastbytes = _mm512_add_epi16(mask_ffffffff,

indexofsecondlastbytes); // indices of the second last bytes

__m512i thirdlastbyte = _mm512_maskz_mov_epi8(m34,

__m512i thirdlastbytes = _mm512_maskz_permutexvar_epi8(0x5555555555555555, indexofthirdlastbytes,

thirdlastbyte); // the third last bytes (of three byte sequences, hi

// surrogate)

}

// Fast path 2: all ASCII or 2 byte

__mmask64 continuation_or_ascii = (tail == SIMDUTF_FULL) ? _knot_mask64(m234) : _kand_mask64(_knot_mask64(m234), b);

// continuation byte tags

__m512i leading2byte = _mm512_maskz_sub_epi8(m234, input, mask_c2c2c2c2);

__mmask64 leading = tail == (tail == SIMDUTF_FULL) ? _kor_mask64(m1, m234) : _kand_mask64(_kor_mask64(m1, m234), b); // first bytes of each sequence

stored at separate 32-bit lanes.

For each lane we have also a character class (`char_class), given in form

corresponding bytes during pshufb.

*/

simdutf_really_inline __m512i expanded_utf8_to_utf32(__m512i char_class, __m512i utf8) {

// _mm512_storeu_si512((__m512i *)latin_output, output); I tried using

// _mm512_storeu_si512 and have the next process_block start from the

// "written_out" point but the compiler shuffles memory in such a way that it

// ****************************

_mm512_mask_storeu_epi8((__m512i *)latin_output, store_mask, output);

}

simdutf_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {

}

/* begin file src/haswell/avx2_detect_encodings.cpp */

// 8 bytes when calling convert_masked_utf8_to_utf32. If you skip the last 16 bytes,

// and if the data is valid, then it is entirely safe because 16 UTF-8 bytes generate

// much more than 8 bytes. However, you cannot generally assume that you have valid

// to give us a good margin.

size_t leading_byte = 0;

size_t margin = size;

leading_byte += (int8_t(in[margin-1]) > -65);

}

// If the input is long enough, then we have that margin-1 is the fourth last leading byte.

// 8 bytes when calling convert_masked_utf8_to_utf32. If you skip the last 16 bytes,

// and if the data is valid, then it is entirely safe because 16 UTF-8 bytes generate

// much more than 8 bytes. However, you cannot generally assume that you have valid

// to give us a good margin.

size_t leading_byte = 0;

size_t margin = size;

leading_byte += (int8_t(in[margin-1]) > -65);

}

// If the input is long enough, then we have that margin-1 is the fourth last leading byte.

}

simdutf_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {

// Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.

}

} // unnamed namespace

// 8 bytes when calling convert_masked_utf8_to_utf32. If you skip the last 16 bytes,

// and if the data is valid, then it is entirely safe because 16 UTF-8 bytes generate

// much more than 8 bytes. However, you cannot generally assume that you have valid

// to give us a good margin.

size_t leading_byte = 0;

size_t margin = size;

leading_byte += (int8_t(in[margin-1]) > -65);

}

// If the input is long enough, then we have that margin-1 is the fourth last leading byte.

// 8 bytes when calling convert_masked_utf8_to_utf32. If you skip the last 16 bytes,

// and if the data is valid, then it is entirely safe because 16 UTF-8 bytes generate

// much more than 8 bytes. However, you cannot generally assume that you have valid

// to give us a good margin.

size_t leading_byte = 0;

size_t margin = size;

leading_byte += (int8_t(in[margin-1]) > -65);

}

// If the input is long enough, then we have that margin-1 is the fourth last leading byte.

}

simdutf_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {

}

/* begin file src/westmere/internal/loader.cpp */

// 8 bytes when calling convert_masked_utf8_to_utf32. If you skip the last 16 bytes,

// and if the data is valid, then it is entirely safe because 16 UTF-8 bytes generate

// much more than 8 bytes. However, you cannot generally assume that you have valid

// to give us a good margin.

size_t leading_byte = 0;

size_t margin = size;

leading_byte += (int8_t(in[margin-1]) > -65);

}

// If the input is long enough, then we have that margin-1 is the fourth last leading byte.

// 8 bytes when calling convert_masked_utf8_to_utf32. If you skip the last 16 bytes,

// and if the data is valid, then it is entirely safe because 16 UTF-8 bytes generate

// much more than 8 bytes. However, you cannot generally assume that you have valid

// to give us a good margin.

size_t leading_byte = 0;

size_t margin = size;

leading_byte += (int8_t(in[margin-1]) > -65);

}

// If the input is long enough, then we have that margin-1 is the fourth last leading byte.