#ifndef LLVM_LIBC_SRC___SUPPORT_FPUTIL_X86_64_FENVIMPL_H
#define LLVM_LIBC_SRC___SUPPORT_FPUTIL_X86_64_FENVIMPL_H
#include "src/__support/macros/attributes.h"
#include "src/__support/macros/config.h"
#include "src/__support/macros/properties/architectures.h"
#if !defined(LIBC_TARGET_ARCH_IS_X86)
#error "Invalid include"
#endif
#include <stdint.h>
#include "hdr/types/fenv_t.h"
#include "src/__support/macros/sanitizer.h"
namespace LIBC_NAMESPACE_DECL {
namespace fputil {
namespace internal {
struct RoundingControlValue {
static constexpr uint16_t TO_NEAREST = 0x0;
static constexpr uint16_t DOWNWARD = 0x1;
static constexpr uint16_t UPWARD = 0x2;
static constexpr uint16_t TOWARD_ZERO = 0x3;
};
static constexpr uint16_t X87_ROUNDING_CONTROL_BIT_POSITION = 10;
static constexpr uint16_t MXCSR_ROUNDING_CONTROL_BIT_POSITION = 13;
struct ExceptionFlags {
static constexpr uint16_t INVALID_F = 0x1;
static constexpr uint16_t DENORMAL_F = 0x2;
static constexpr uint16_t DIV_BY_ZERO_F = 0x4;
static constexpr uint16_t OVERFLOW_F = 0x8;
static constexpr uint16_t UNDERFLOW_F = 0x10;
static constexpr uint16_t INEXACT_F = 0x20;
};
static constexpr uint16_t X87_EXCEPTION_CONTROL_BIT_POSITION = 0;
static constexpr uint16_t X87_EXCEPTION_CONTROL_BIT_POSITION_HIGH = 24;
static constexpr uint16_t MXCSR_EXCEPTION_CONTOL_BIT_POISTION = 7;
LIBC_INLINE uint16_t get_status_value_for_except(int excepts) {
return ((excepts & FE_INVALID) ? ExceptionFlags::INVALID_F : 0) |
#ifdef __FE_DENORM
((excepts & __FE_DENORM) ? ExceptionFlags::DENORMAL_F : 0) |
#endif
((excepts & FE_DIVBYZERO) ? ExceptionFlags::DIV_BY_ZERO_F : 0) |
((excepts & FE_OVERFLOW) ? ExceptionFlags::OVERFLOW_F : 0) |
((excepts & FE_UNDERFLOW) ? ExceptionFlags::UNDERFLOW_F : 0) |
((excepts & FE_INEXACT) ? ExceptionFlags::INEXACT_F : 0);
}
LIBC_INLINE int exception_status_to_macro(uint16_t status) {
return ((status & ExceptionFlags::INVALID_F) ? FE_INVALID : 0) |
#ifdef __FE_DENORM
((status & ExceptionFlags::DENORMAL_F) ? __FE_DENORM : 0) |
#endif
((status & ExceptionFlags::DIV_BY_ZERO_F) ? FE_DIVBYZERO : 0) |
((status & ExceptionFlags::OVERFLOW_F) ? FE_OVERFLOW : 0) |
((status & ExceptionFlags::UNDERFLOW_F) ? FE_UNDERFLOW : 0) |
((status & ExceptionFlags::INEXACT_F) ? FE_INEXACT : 0);
}
struct X87StateDescriptor {
uint16_t control_word;
uint16_t unused1;
uint16_t status_word;
uint16_t unused2;
uint32_t _[5];
};
LIBC_INLINE uint16_t get_x87_control_word() {
uint16_t w;
__asm__ __volatile__("fnstcw %0" : "=m"(w)::);
MSAN_UNPOISON(&w, sizeof(w));
return w;
}
LIBC_INLINE void write_x87_control_word(uint16_t w) {
__asm__ __volatile__("fldcw %0" : : "m"(w) :);
}
LIBC_INLINE uint16_t get_x87_status_word() {
uint16_t w;
__asm__ __volatile__("fnstsw %0" : "=m"(w)::);
MSAN_UNPOISON(&w, sizeof(w));
return w;
}
LIBC_INLINE void clear_x87_exceptions() {
__asm__ __volatile__("fnclex" : : :);
}
LIBC_INLINE uint32_t get_mxcsr() {
uint32_t w;
__asm__ __volatile__("stmxcsr %0" : "=m"(w)::);
MSAN_UNPOISON(&w, sizeof(w));
return w;
}
LIBC_INLINE void write_mxcsr(uint32_t w) {
__asm__ __volatile__("ldmxcsr %0" : : "m"(w) :);
}
LIBC_INLINE void get_x87_state_descriptor(X87StateDescriptor &s) {
__asm__ __volatile__("fnstenv %0" : "=m"(s));
MSAN_UNPOISON(&s, sizeof(s));
}
LIBC_INLINE void write_x87_state_descriptor(const X87StateDescriptor &s) {
__asm__ __volatile__("fldenv %0" : : "m"(s) :);
}
LIBC_INLINE void fwait() { __asm__ __volatile__("fwait"); }
}
LIBC_INLINE int enable_except(int excepts) {
uint16_t bit_mask = internal::get_status_value_for_except(excepts);
uint16_t x87_cw = internal::get_x87_control_word();
uint16_t old_excepts = ~x87_cw & 0x3F;
x87_cw &= ~bit_mask;
internal::write_x87_control_word(x87_cw);
uint32_t mxcsr = internal::get_mxcsr();
mxcsr &= ~(bit_mask << internal::MXCSR_EXCEPTION_CONTOL_BIT_POISTION);
internal::write_mxcsr(mxcsr);
return internal::exception_status_to_macro(old_excepts);
}
LIBC_INLINE int disable_except(int excepts) {
uint16_t bit_mask = internal::get_status_value_for_except(excepts);
uint16_t x87_cw = internal::get_x87_control_word();
uint16_t old_excepts = ~x87_cw & 0x3F;
x87_cw |= bit_mask;
internal::write_x87_control_word(x87_cw);
uint32_t mxcsr = internal::get_mxcsr();
mxcsr |= (bit_mask << internal::MXCSR_EXCEPTION_CONTOL_BIT_POISTION);
internal::write_mxcsr(mxcsr);
return internal::exception_status_to_macro(old_excepts);
}
LIBC_INLINE int get_except() {
uint16_t mxcsr = static_cast<uint16_t>(internal::get_mxcsr());
uint16_t enabled_excepts = ~(mxcsr >> 7) & 0x3F;
return internal::exception_status_to_macro(enabled_excepts);
}
LIBC_INLINE int clear_except(int excepts) {
internal::X87StateDescriptor state;
internal::get_x87_state_descriptor(state);
state.status_word &=
static_cast<uint16_t>(~internal::get_status_value_for_except(excepts));
internal::write_x87_state_descriptor(state);
uint32_t mxcsr = internal::get_mxcsr();
mxcsr &= ~internal::get_status_value_for_except(excepts);
internal::write_mxcsr(mxcsr);
return 0;
}
LIBC_INLINE int test_except(int excepts) {
uint16_t status_word = internal::get_x87_status_word();
uint32_t mxcsr = internal::get_mxcsr();
uint16_t status_value = internal::get_status_value_for_except(excepts);
return internal::exception_status_to_macro(
static_cast<uint16_t>(status_value & (status_word | mxcsr)));
}
LIBC_INLINE int set_except(int excepts) {
uint16_t status_value = internal::get_status_value_for_except(excepts);
internal::X87StateDescriptor state;
internal::get_x87_state_descriptor(state);
state.status_word |= status_value;
internal::write_x87_state_descriptor(state);
uint32_t mxcsr = internal::get_mxcsr();
mxcsr |= status_value;
internal::write_mxcsr(mxcsr);
return 0;
}
LIBC_INLINE int raise_except(int excepts) {
uint16_t status_value = internal::get_status_value_for_except(excepts);
auto raise_helper = [](uint16_t singleExceptFlag) {
internal::X87StateDescriptor state;
uint32_t mxcsr = 0;
internal::get_x87_state_descriptor(state);
mxcsr = internal::get_mxcsr();
state.status_word |= singleExceptFlag;
mxcsr |= singleExceptFlag;
internal::write_x87_state_descriptor(state);
internal::write_mxcsr(mxcsr);
internal::fwait();
};
if (status_value & internal::ExceptionFlags::INVALID_F)
raise_helper(internal::ExceptionFlags::INVALID_F);
if (status_value & internal::ExceptionFlags::DIV_BY_ZERO_F)
raise_helper(internal::ExceptionFlags::DIV_BY_ZERO_F);
if (status_value & internal::ExceptionFlags::OVERFLOW_F)
raise_helper(internal::ExceptionFlags::OVERFLOW_F);
if (status_value & internal::ExceptionFlags::UNDERFLOW_F)
raise_helper(internal::ExceptionFlags::UNDERFLOW_F);
if (status_value & internal::ExceptionFlags::INEXACT_F)
raise_helper(internal::ExceptionFlags::INEXACT_F);
#ifdef __FE_DENORM
if (status_value & internal::ExceptionFlags::DENORMAL_F) {
raise_helper(internal::ExceptionFlags::DENORMAL_F);
}
#endif
return 0;
}
LIBC_INLINE int get_round() {
uint16_t bit_value =
(internal::get_mxcsr() >> internal::MXCSR_ROUNDING_CONTROL_BIT_POSITION) &
0x3;
switch (bit_value) {
case internal::RoundingControlValue::TO_NEAREST:
return FE_TONEAREST;
case internal::RoundingControlValue::DOWNWARD:
return FE_DOWNWARD;
case internal::RoundingControlValue::UPWARD:
return FE_UPWARD;
case internal::RoundingControlValue::TOWARD_ZERO:
return FE_TOWARDZERO;
default:
return -1;
}
}
LIBC_INLINE int set_round(int mode) {
uint16_t bit_value;
switch (mode) {
case FE_TONEAREST:
bit_value = internal::RoundingControlValue::TO_NEAREST;
break;
case FE_DOWNWARD:
bit_value = internal::RoundingControlValue::DOWNWARD;
break;
case FE_UPWARD:
bit_value = internal::RoundingControlValue::UPWARD;
break;
case FE_TOWARDZERO:
bit_value = internal::RoundingControlValue::TOWARD_ZERO;
break;
default:
return 1;
}
uint16_t x87_value = static_cast<uint16_t>(
bit_value << internal::X87_ROUNDING_CONTROL_BIT_POSITION);
uint16_t x87_control = internal::get_x87_control_word();
x87_control = static_cast<uint16_t>(
(x87_control &
~(uint16_t(0x3) << internal::X87_ROUNDING_CONTROL_BIT_POSITION)) |
x87_value);
internal::write_x87_control_word(x87_control);
uint32_t mxcsr_value = bit_value
<< internal::MXCSR_ROUNDING_CONTROL_BIT_POSITION;
uint32_t mxcsr_control = internal::get_mxcsr();
mxcsr_control = (mxcsr_control &
~(0x3 << internal::MXCSR_ROUNDING_CONTROL_BIT_POSITION)) |
mxcsr_value;
internal::write_mxcsr(mxcsr_control);
return 0;
}
namespace internal {
#if defined(_WIN32)
struct FPState {
uint32_t control_word;
uint32_t status_word;
};
#elif defined(__APPLE__)
struct FPState {
uint16_t control_word;
uint16_t status_word;
uint32_t mxcsr;
uint8_t reserved[8];
};
#else
struct FPState {
X87StateDescriptor x87_status;
uint32_t mxcsr;
};
#endif
}
static_assert(
sizeof(fenv_t) == sizeof(internal::FPState),
"Internal floating point state does not match the public fenv_t type.");
#ifdef _WIN32
struct WinExceptionFlags {
static constexpr uint32_t INEXACT_WIN = 0x01;
static constexpr uint32_t UNDERFLOW_WIN = 0x02;
static constexpr uint32_t OVERFLOW_WIN = 0x04;
static constexpr uint32_t DIV_BY_ZERO_WIN = 0x08;
static constexpr uint32_t INVALID_WIN = 0x10;
static constexpr uint32_t DENORMAL_WIN = 0x20;
static constexpr uint32_t HIGH_OFFSET = 24;
static constexpr uint32_t HIGH_INEXACT = INEXACT_WIN << HIGH_OFFSET;
static constexpr uint32_t HIGH_UNDERFLOW = UNDERFLOW_WIN << HIGH_OFFSET;
static constexpr uint32_t HIGH_OVERFLOW = OVERFLOW_WIN << HIGH_OFFSET;
static constexpr uint32_t HIGH_DIV_BY_ZERO = DIV_BY_ZERO_WIN << HIGH_OFFSET;
static constexpr uint32_t HIGH_INVALID = INVALID_WIN << HIGH_OFFSET;
static constexpr uint32_t HIGH_DENORMAL = DENORMAL_WIN << HIGH_OFFSET;
};
fenv_t control word format:
Windows (at least for x64) uses a 4 byte control fenv control word stored in
a 32 bit integer. The first byte contains just the rounding mode and the
exception masks, while the last two bytes contain that same information as
well as the flush-to-zero and denormals-are-zero flags. The flags are
represented with a truth table:
00 - No flags set
01 - Flush-to-zero and Denormals-are-zero set
11 - Flush-to-zero set
10 - Denormals-are-zero set
U represents unused.
+-----Rounding Mode-----+
| |
++ ++
|| ||
RRMMMMMM UUUUUUUU UUUUFFRR UUMMMMMM
| | || | |
+----+ flags---++ +----+
| |
+------Exception Masks-----+
fenv_t status word format:
The status word is a lot simpler for this conversion, since only the
exception flags are used in the MXCSR.
+----+---Exception Flags---+----+
| | | |
UUEEEEEE UUUUUUUU UUUUUUUU UUEEEEEE
MXCSR Format:
The MXCSR format is the same information, just organized differently. Since
the fenv_t struct for windows doesn't include the mxcsr bits, they must be
generated from the control word bits.
Exception Masks---+ +---Exception Flags
| |
Flush-to-zero---+ +----+ +----+
| | | | |
FRRMMMMMMDEEEEEE
|| |
++ +---Denormals-are-zero
|
+---Rounding Mode
The mask and flag order is as follows:
fenv_t mxcsr
denormal inexact
invalid underflow
div by 0 overflow
overflow div by 0
underflow denormal
inexact invalid
This is almost reverse, except for denormal and invalid which are in the
same order in both.
*/
LIBC_INLINE int get_env(fenv_t *envp) {
internal::FPState *state = reinterpret_cast<internal::FPState *>(envp);
uint32_t status_word = 0;
uint32_t control_word = 0;
uint32_t mxcsr = internal::get_mxcsr();
status_word |= (mxcsr & (internal::ExceptionFlags::INVALID_F |
internal::ExceptionFlags::DENORMAL_F))
<< 4;
status_word |= (mxcsr & internal::ExceptionFlags::DIV_BY_ZERO_F) << 1;
status_word |= (mxcsr & internal::ExceptionFlags::OVERFLOW_F) >> 1;
status_word |= (mxcsr & internal::ExceptionFlags::UNDERFLOW_F) >> 3;
status_word |= (mxcsr & internal::ExceptionFlags::INEXACT_F) >> 5;
status_word |= status_word << WinExceptionFlags::HIGH_OFFSET;
control_word |= (mxcsr & ((internal::ExceptionFlags::INVALID_F |
internal::ExceptionFlags::DENORMAL_F)
<< 7)) >>
3;
control_word |= (mxcsr & (internal::ExceptionFlags::DIV_BY_ZERO_F << 7)) >> 6;
control_word |= (mxcsr & (internal::ExceptionFlags::OVERFLOW_F << 7)) >> 8;
control_word |= (mxcsr & (internal::ExceptionFlags::UNDERFLOW_F << 7)) >> 10;
control_word |= (mxcsr & (internal::ExceptionFlags::INEXACT_F << 7)) >> 12;
control_word |= control_word << WinExceptionFlags::HIGH_OFFSET;
control_word |= (mxcsr & 0x6000) >> 5;
control_word |= (mxcsr & 0x6000) << 17;
control_word |= (mxcsr & 0x8000) >> 5;
control_word |= (((mxcsr & 0x8000) >> 9) ^ (mxcsr & 0x0040)) << 5;
state->control_word = control_word;
state->status_word = status_word;
return 0;
}
LIBC_INLINE int set_env(const fenv_t *envp) {
const internal::FPState *state =
reinterpret_cast<const internal::FPState *>(envp);
uint32_t mxcsr = 0;
mxcsr |= static_cast<uint16_t>(
(state->status_word &
(WinExceptionFlags::HIGH_DENORMAL | WinExceptionFlags::HIGH_INVALID)) >>
28);
mxcsr |= static_cast<uint16_t>(
(state->status_word & WinExceptionFlags::HIGH_DIV_BY_ZERO) >> 25);
mxcsr |= static_cast<uint16_t>(
(state->status_word & WinExceptionFlags::HIGH_OVERFLOW) >> 23);
mxcsr |= static_cast<uint16_t>(
(state->status_word & WinExceptionFlags::HIGH_UNDERFLOW) >> 21);
mxcsr |= static_cast<uint16_t>(
(state->status_word & WinExceptionFlags::HIGH_INEXACT) >> 19);
mxcsr |= static_cast<uint16_t>(
(((state->control_word & 0x800) >> 1) ^ (state->control_word & 0x400)) >>
4);
mxcsr |= static_cast<uint16_t>(
(state->control_word &
(WinExceptionFlags::HIGH_DENORMAL | WinExceptionFlags::HIGH_INVALID)) >>
21);
mxcsr |= static_cast<uint16_t>(
(state->control_word & WinExceptionFlags::HIGH_DIV_BY_ZERO) >> 18);
mxcsr |= static_cast<uint16_t>(
(state->control_word & WinExceptionFlags::HIGH_OVERFLOW) >> 16);
mxcsr |= static_cast<uint16_t>(
(state->control_word & WinExceptionFlags::HIGH_UNDERFLOW) >> 14);
mxcsr |= static_cast<uint16_t>(
(state->control_word & WinExceptionFlags::HIGH_INEXACT) >> 12);
mxcsr |= static_cast<uint16_t>((state->control_word & 0xc0000000) >> 17);
mxcsr |= static_cast<uint16_t>((state->control_word & 0x400) << 5);
internal::write_mxcsr(mxcsr);
return 0;
}
#else
LIBC_INLINE int get_env(fenv_t *envp) {
internal::FPState *state = reinterpret_cast<internal::FPState *>(envp);
#ifdef __APPLE__
internal::X87StateDescriptor x87_status;
internal::get_x87_state_descriptor(x87_status);
state->control_word = x87_status.control_word;
state->status_word = x87_status.status_word;
#else
internal::get_x87_state_descriptor(state->x87_status);
#endif
state->mxcsr = internal::get_mxcsr();
return 0;
}
LIBC_INLINE int set_env(const fenv_t *envp) {
internal::X87StateDescriptor x87_status;
internal::get_x87_state_descriptor(x87_status);
if (envp == FE_DFL_ENV) {
x87_status.status_word &= ~uint16_t(0x3F);
for (int i = 0; i < 5; i++)
x87_status._[i] = 0;
x87_status.control_word |= uint16_t(0x3F);
x87_status.control_word &= ~(uint16_t(0x3) << 10);
x87_status.control_word |= (uint16_t(0x3) << 8);
internal::write_x87_state_descriptor(x87_status);
uint32_t mxcsr = internal::get_mxcsr();
mxcsr &= ~uint16_t(0x3F);
mxcsr &= ~(uint16_t(0x1) << 6);
mxcsr |= (uint16_t(0x3F) << 7);
mxcsr &= ~(uint16_t(0x3) << 13);
mxcsr &= ~(uint16_t(0x1) << 15);
internal::write_mxcsr(mxcsr);
return 0;
}
const internal::FPState *fpstate =
reinterpret_cast<const internal::FPState *>(envp);
x87_status.status_word &= ~uint16_t(0x3F);
#ifdef __APPLE__
x87_status.status_word |= (fpstate->status_word & 0x3F);
x87_status.control_word = fpstate->control_word;
#else
x87_status.status_word |= (fpstate->x87_status.status_word & 0x3F);
for (int i = 0; i < 5; i++)
x87_status._[i] = fpstate->x87_status._[i];
x87_status.control_word = fpstate->x87_status.control_word;
#endif
internal::write_x87_state_descriptor(x87_status);
internal::write_mxcsr(fpstate->mxcsr);
return 0;
}
#endif
}
}
#endif