//! Provides CRC-32/ISCSI and CRC-32/ISO-HDLC calculations using a fusion of native CLMUL
//! instructions and native CRC calculation instructions on x86_64.
//!
//! https://www.corsix.org/content/fast-crc32c-4k
//! https://www.corsix.org/content/alternative-exposition-crc32_4k_pclmulqdq
//!
//! Converted to Rust from the original C code generated by https://github.com/corsix/fast-crc32/
//! with the help of Claude.ai using:
//!
//! ./generate -i avx512_vpclmulqdq -p crc32c -a v3x2
//! ./generate -i avx512 -p crc32c -a v4s3x3
//! ./generate -i sse -p crc32c -a v4s3x3
//!
//! Modified as necessary for this Rust implementation.
//!
//! MIT licensed.

#![cfg(target_arch = "x86_64")]

use std::arch::x86_64::*;

/// Safe wrapper for CRC32 iSCSI calculation using AVX-512
//#[rustversion::before(1.89)]
#[inline(always)]
#[cfg(not(feature = "vpclmulqdq"))]
pub fn crc32_iscsi(crc: u32, data: &[u8]) -> u32 {
    unsafe { crc32_iscsi_sse_v4s3x3(crc, data.as_ptr(), data.len()) }
}

//#[rustversion::since(1.89)]
#[inline(always)]
#[cfg(feature = "vpclmulqdq")]
pub fn crc32_iscsi(crc: u32, data: &[u8]) -> u32 {
    if is_x86_feature_detected!("vpclmulqdq")
        && is_x86_feature_detected!("avx512f")
        && is_x86_feature_detected!("avx512vl")
    {
        unsafe {
            return crc32_iscsi_avx512_vpclmulqdq_v3x2(crc, data.as_ptr(), data.len());
        }
    }

    if is_x86_feature_detected!("avx512f") && is_x86_feature_detected!("avx512vl") {
        unsafe {
            return crc32_iscsi_avx512_v4s3x3(crc, data.as_ptr(), data.len());
        }
    }

    unsafe { crc32_iscsi_sse_v4s3x3(crc, data.as_ptr(), data.len()) }
}

//#[rustversion::since(1.89)]
#[inline]
#[cfg(feature = "vpclmulqdq")]
#[target_feature(enable = "avx512f,avx512vl,vpclmulqdq")]
unsafe fn clmul_lo_avx512_vpclmulqdq(a: __m512i, b: __m512i) -> __m512i {
    _mm512_clmulepi64_epi128(a, b, 0)
}

//#[rustversion::since(1.89)]
#[inline]
#[cfg(feature = "vpclmulqdq")]
#[target_feature(enable = "avx512f,avx512vl,vpclmulqdq")]
unsafe fn clmul_hi_avx512_vpclmulqdq(a: __m512i, b: __m512i) -> __m512i {
    _mm512_clmulepi64_epi128(a, b, 17)
}

#[inline]
#[target_feature(enable = "pclmulqdq")]
unsafe fn clmul_lo_sse(a: __m128i, b: __m128i) -> __m128i {
    _mm_clmulepi64_si128(a, b, 0)
}

#[inline]
#[target_feature(enable = "pclmulqdq")]
unsafe fn clmul_hi_sse(a: __m128i, b: __m128i) -> __m128i {
    _mm_clmulepi64_si128(a, b, 17)
}

#[inline]
#[target_feature(enable = "pclmulqdq")]
unsafe fn clmul_scalar_sse(a: u32, b: u32) -> __m128i {
    _mm_clmulepi64_si128(_mm_cvtsi32_si128(a as i32), _mm_cvtsi32_si128(b as i32), 0)
}

// x^n mod P, in log(n) time
#[target_feature(enable = "sse4.2,pclmulqdq")]
unsafe fn xnmodp_iscsi_sse(mut n: u64) -> u32 {
    let mut stack = !1u64;
    let mut acc: u32;
    let mut low: u32;

    while n > 191 {
        stack = (stack << 1) + (n & 1);
        n = (n >> 1) - 16;
    }
    stack = !stack;
    acc = 0x80000000u32 >> (n & 31);
    n >>= 5;

    while n > 0 {
        // Use hardware CRC32C instruction
        acc = _mm_crc32_u32(acc, 0);
        n -= 1;
    }

    while {
        low = (stack & 1) as u32;
        stack >>= 1;
        stack != 0
    } {
        let x = _mm_cvtsi32_si128(acc as i32);
        let y = _mm_cvtsi128_si64(_mm_clmulepi64_si128(x, x, 0)) as u64;
        acc = _mm_crc32_u64(0, y << low) as u32;
    }
    acc
}

#[inline]
#[target_feature(enable = "pclmulqdq")]
unsafe fn crc_shift_iscsi_sse(crc: u32, nbytes: usize) -> __m128i {
    clmul_scalar_sse(crc, xnmodp_iscsi_sse((nbytes * 8 - 33) as u64))
}

#[inline]
#[target_feature(enable = "sse4.1")]
unsafe fn mm_extract_epi64(val: __m128i, idx: i32) -> u64 {
    if idx == 0 {
        _mm_cvtsi128_si64(val) as u64
    } else {
        _mm_cvtsi128_si64(_mm_srli_si128(val, 8)) as u64
    }
}

#[inline]
#[target_feature(enable = "sse4.2")]
unsafe fn mm_crc32_u64(crc: u32, val: u64) -> u32 {
    _mm_crc32_u64(crc.into(), val) as u32
}

/// Converted to Rust from the original C code generated by https://github.com/corsix/fast-crc32/
/// using:
///
/// ./generate -i avx512_vpclmulqdq -p crc32c -a v3x2
//#[rustversion::since(1.89)]
#[inline]
#[cfg(feature = "vpclmulqdq")]
#[target_feature(enable = "avx512f,avx512vl,vpclmulqdq,sse4.2")]
pub unsafe fn crc32_iscsi_avx512_vpclmulqdq_v3x2(
    mut crc0: u32,
    mut buf: *const u8,
    mut len: usize,
) -> u32 {
    // Align to 8-byte boundary
    while len > 0 && (buf as usize & 7) != 0 {
        crc0 = _mm_crc32_u8(crc0, *buf);
        buf = buf.add(1);
        len -= 1;
    }

    // Align to 64-byte boundary (cache line)
    while (buf as usize & 56) != 0 && len >= 8 {
        crc0 = _mm_crc32_u64(crc0.into(), *(buf as *const u64)) as u32;
        buf = buf.add(8);
        len -= 8;
    }

    if len >= 384 {
        // First vector chunk - load three 512-bit vectors (192 bytes total)
        let mut x0 = _mm512_loadu_si512(buf as *const __m512i);
        let mut x1 = _mm512_loadu_si512(buf.add(64) as *const __m512i);
        let mut x2 = _mm512_loadu_si512(buf.add(128) as *const __m512i);

        // Create the multiplication constant vector
        // Pattern: [0xa87ab8a8, 0, 0xab7aff2a, 0] repeated across all 128-bit lanes
        let k_128 = _mm_setr_epi32(0xa87ab8a8u32 as i32, 0, 0xab7aff2au32 as i32, 0);
        let mut k = _mm512_broadcast_i32x4(k_128);

        // XOR the CRC into the first vector's low 32 bits
        let crc_vec = _mm512_castsi128_si512(_mm_cvtsi32_si128(crc0 as i32));
        x0 = _mm512_xor_si512(crc_vec, x0);

        // First round of polynomial multiplication
        let mut y0 = clmul_lo_avx512_vpclmulqdq(x0, k);
        x0 = clmul_hi_avx512_vpclmulqdq(x0, k);
        let mut y1 = clmul_lo_avx512_vpclmulqdq(x1, k);
        x1 = clmul_hi_avx512_vpclmulqdq(x1, k);
        let mut y2 = clmul_lo_avx512_vpclmulqdq(x2, k);
        x2 = clmul_hi_avx512_vpclmulqdq(x2, k);

        // XOR with next chunk of data using ternary logic (A XOR B XOR C)
        // 0x96 = A XOR B XOR C in ternary logic notation
        x0 = _mm512_ternarylogic_epi64(
            x0,
            y0,
            _mm512_loadu_si512(buf.add(192) as *const __m512i),
            0x96,
        );
        x1 = _mm512_ternarylogic_epi64(
            x1,
            y1,
            _mm512_loadu_si512(buf.add(256) as *const __m512i),
            0x96,
        );
        x2 = _mm512_ternarylogic_epi64(
            x2,
            y2,
            _mm512_loadu_si512(buf.add(320) as *const __m512i),
            0x96,
        );

        buf = buf.add(384);
        len -= 384;

        // Main loop - process 384 bytes at a time
        while len >= 384 {
            // First folding step
            y0 = clmul_lo_avx512_vpclmulqdq(x0, k);
            x0 = clmul_hi_avx512_vpclmulqdq(x0, k);
            y1 = clmul_lo_avx512_vpclmulqdq(x1, k);
            x1 = clmul_hi_avx512_vpclmulqdq(x1, k);
            y2 = clmul_lo_avx512_vpclmulqdq(x2, k);
            x2 = clmul_hi_avx512_vpclmulqdq(x2, k);

            x0 = _mm512_ternarylogic_epi64(x0, y0, _mm512_loadu_si512(buf as *const __m512i), 0x96);
            x1 = _mm512_ternarylogic_epi64(
                x1,
                y1,
                _mm512_loadu_si512(buf.add(64) as *const __m512i),
                0x96,
            );
            x2 = _mm512_ternarylogic_epi64(
                x2,
                y2,
                _mm512_loadu_si512(buf.add(128) as *const __m512i),
                0x96,
            );

            // Second folding step
            y0 = clmul_lo_avx512_vpclmulqdq(x0, k);
            x0 = clmul_hi_avx512_vpclmulqdq(x0, k);
            y1 = clmul_lo_avx512_vpclmulqdq(x1, k);
            x1 = clmul_hi_avx512_vpclmulqdq(x1, k);
            y2 = clmul_lo_avx512_vpclmulqdq(x2, k);
            x2 = clmul_hi_avx512_vpclmulqdq(x2, k);

            x0 = _mm512_ternarylogic_epi64(
                x0,
                y0,
                _mm512_loadu_si512(buf.add(192) as *const __m512i),
                0x96,
            );
            x1 = _mm512_ternarylogic_epi64(
                x1,
                y1,
                _mm512_loadu_si512(buf.add(256) as *const __m512i),
                0x96,
            );
            x2 = _mm512_ternarylogic_epi64(
                x2,
                y2,
                _mm512_loadu_si512(buf.add(320) as *const __m512i),
                0x96,
            );

            buf = buf.add(384);
            len -= 384;
        }

        // Reduce x0, x1, x2 to just x0
        let k_128 = _mm_setr_epi32(0x740eef02u32 as i32, 0, 0x9e4addf8u32 as i32, 0);
        k = _mm512_broadcast_i32x4(k_128);

        y0 = clmul_lo_avx512_vpclmulqdq(x0, k);
        x0 = clmul_hi_avx512_vpclmulqdq(x0, k);
        x0 = _mm512_ternarylogic_epi64(x0, y0, x1, 0x96);
        x1 = x2;

        y0 = clmul_lo_avx512_vpclmulqdq(x0, k);
        x0 = clmul_hi_avx512_vpclmulqdq(x0, k);
        x0 = _mm512_ternarylogic_epi64(x0, y0, x1, 0x96);

        // Reduce 512 bits to 128 bits
        // Multiple reduction constants for different parts of the 512-bit vector
        k = _mm512_setr_epi32(
            0x1c291d04u32 as i32,
            0,
            0xddc0152bu32 as i32,
            0, // Lane 0
            0x3da6d0cbu32 as i32,
            0,
            0xba4fc28eu32 as i32,
            0, // Lane 1
            0xf20c0dfeu32 as i32,
            0,
            0x493c7d27u32 as i32,
            0, // Lane 2
            0,
            0,
            0,
            0, // Lane 3 (unused)
        );

        y0 = clmul_lo_avx512_vpclmulqdq(x0, k);
        k = clmul_hi_avx512_vpclmulqdq(x0, k);
        y0 = _mm512_xor_si512(y0, k);

        // Extract 128-bit lanes and combine them
        let lane0 = _mm512_castsi512_si128(y0);
        let lane1 = _mm512_extracti32x4_epi32(y0, 1);
        let lane2 = _mm512_extracti32x4_epi32(y0, 2);
        let lane3 = _mm512_extracti32x4_epi32(x0, 3);

        // Combine all lanes using ternary logic
        let mut z0 = _mm_ternarylogic_epi64(lane0, lane1, lane2, 0x96);
        z0 = _mm_xor_si128(z0, lane3);

        // Reduce 128 bits to 32 bits using CRC32 instructions
        crc0 = _mm_crc32_u64(0, _mm_extract_epi64(z0, 0) as u64) as u32;
        crc0 = _mm_crc32_u64(crc0.into(), _mm_extract_epi64(z0, 1) as u64) as u32;
    }

    // Process remaining 8-byte chunks
    while len >= 8 {
        crc0 = _mm_crc32_u64(crc0.into(), *(buf as *const u64)) as u32;
        buf = buf.add(8);
        len -= 8;
    }

    // Process remaining bytes
    while len > 0 {
        crc0 = _mm_crc32_u8(crc0, *buf);
        buf = buf.add(1);
        len -= 1;
    }

    crc0
}

/// Converted to Rust from the original C code generated by https://github.com/corsix/fast-crc32/
/// using:
///
/// ./generate -i avx512 -p crc32c -a v4s3x3
//#[rustversion::since(1.89)]
#[inline]
#[cfg(feature = "vpclmulqdq")]
#[target_feature(enable = "avx2,avx512f,avx512vl,pclmulqdq")]
pub unsafe fn crc32_iscsi_avx512_v4s3x3(mut crc0: u32, mut buf: *const u8, mut len: usize) -> u32 {
    // Align to 8-byte boundary using hardware CRC32C instructions
    while len > 0 && (buf as usize & 7) != 0 {
        crc0 = _mm_crc32_u8(crc0, *buf);
        buf = buf.add(1);
        len -= 1;
    }

    // Handle 8-byte alignment
    if (buf as usize & 8) != 0 && len >= 8 {
        crc0 = _mm_crc32_u64(crc0.into(), *(buf as *const u64)) as u32;
        buf = buf.add(8);
        len -= 8;
    }

    if len >= 144 {
        let blk = (len - 8) / 136;
        let klen = blk * 24;
        let buf2 = buf;
        let mut crc1 = 0u32;
        let mut crc2 = 0u32;

        // First vector chunk - load four 128-bit vectors (64 bytes total)
        let mut x0 = _mm_loadu_si128(buf2 as *const __m128i);
        let mut x1 = _mm_loadu_si128(buf2.add(16) as *const __m128i);
        let mut x2 = _mm_loadu_si128(buf2.add(32) as *const __m128i);
        let mut x3 = _mm_loadu_si128(buf2.add(48) as *const __m128i);

        // iSCSI-specific folding constant (different from ISO-HDLC)
        let mut k = _mm_setr_epi32(0x740eef02u32 as i32, 0, 0x9e4addf8u32 as i32, 0);

        // XOR the CRC into the first vector's low 32 bits
        x0 = _mm_xor_si128(_mm_cvtsi32_si128(crc0 as i32), x0);
        crc0 = 0;

        let mut buf2 = buf2.add(64);
        len -= 136;
        buf = buf.add(blk * 64);

        // Main loop - process 144 bytes at a time
        while len >= 144 {
            let y0 = clmul_lo_sse(x0, k);
            x0 = clmul_hi_sse(x0, k);
            let y1 = clmul_lo_sse(x1, k);
            x1 = clmul_hi_sse(x1, k);
            let y2 = clmul_lo_sse(x2, k);
            x2 = clmul_hi_sse(x2, k);
            let y3 = clmul_lo_sse(x3, k);
            x3 = clmul_hi_sse(x3, k);

            // XOR with next chunk of data using ternary logic (A XOR B XOR C)
            x0 = _mm_ternarylogic_epi64(x0, y0, _mm_loadu_si128(buf2 as *const __m128i), 0x96);
            x1 = _mm_ternarylogic_epi64(
                x1,
                y1,
                _mm_loadu_si128(buf2.add(16) as *const __m128i),
                0x96,
            );
            x2 = _mm_ternarylogic_epi64(
                x2,
                y2,
                _mm_loadu_si128(buf2.add(32) as *const __m128i),
                0x96,
            );
            x3 = _mm_ternarylogic_epi64(
                x3,
                y3,
                _mm_loadu_si128(buf2.add(48) as *const __m128i),
                0x96,
            );

            // Process scalar data in parallel using hardware CRC32C
            crc0 = _mm_crc32_u64(crc0.into(), *(buf as *const u64)) as u32;
            crc1 = _mm_crc32_u64(crc1.into(), *(buf.add(klen) as *const u64)) as u32;
            crc2 = _mm_crc32_u64(crc2.into(), *(buf.add(klen * 2) as *const u64)) as u32;
            crc0 = _mm_crc32_u64(crc0.into(), *(buf.add(8) as *const u64)) as u32;
            crc1 = _mm_crc32_u64(crc1.into(), *(buf.add(klen + 8) as *const u64)) as u32;
            crc2 = _mm_crc32_u64(crc2.into(), *(buf.add(klen * 2 + 8) as *const u64)) as u32;
            crc0 = _mm_crc32_u64(crc0.into(), *(buf.add(16) as *const u64)) as u32;
            crc1 = _mm_crc32_u64(crc1.into(), *(buf.add(klen + 16) as *const u64)) as u32;
            crc2 = _mm_crc32_u64(crc2.into(), *(buf.add(klen * 2 + 16) as *const u64)) as u32;

            buf = buf.add(24);
            buf2 = buf2.add(64);
            len -= 136;
        }

        // Reduce x0 ... x3 to just x0 using iSCSI-specific constants
        k = _mm_setr_epi32(0xf20c0dfeu32 as i32, 0, 0x493c7d27u32 as i32, 0);

        let y0 = clmul_lo_sse(x0, k);
        x0 = clmul_hi_sse(x0, k);
        let y2 = clmul_lo_sse(x2, k);
        x2 = clmul_hi_sse(x2, k);

        x0 = _mm_ternarylogic_epi64(x0, y0, x1, 0x96);
        x2 = _mm_ternarylogic_epi64(x2, y2, x3, 0x96);

        k = _mm_setr_epi32(0x3da6d0cbu32 as i32, 0, 0xba4fc28eu32 as i32, 0);

        let y0 = clmul_lo_sse(x0, k);
        x0 = clmul_hi_sse(x0, k);
        x0 = _mm_ternarylogic_epi64(x0, y0, x2, 0x96);

        // Final scalar chunk
        crc0 = _mm_crc32_u64(crc0.into(), *(buf as *const u64)) as u32;
        crc1 = _mm_crc32_u64(crc1.into(), *(buf.add(klen) as *const u64)) as u32;
        crc2 = _mm_crc32_u64(crc2.into(), *(buf.add(klen * 2) as *const u64)) as u32;
        crc0 = _mm_crc32_u64(crc0.into(), *(buf.add(8) as *const u64)) as u32;
        crc1 = _mm_crc32_u64(crc1.into(), *(buf.add(klen + 8) as *const u64)) as u32;
        crc2 = _mm_crc32_u64(crc2.into(), *(buf.add(klen * 2 + 8) as *const u64)) as u32;
        crc0 = _mm_crc32_u64(crc0.into(), *(buf.add(16) as *const u64)) as u32;
        crc1 = _mm_crc32_u64(crc1.into(), *(buf.add(klen + 16) as *const u64)) as u32;
        crc2 = _mm_crc32_u64(crc2.into(), *(buf.add(klen * 2 + 16) as *const u64)) as u32;
        buf = buf.add(24);

        let vc0 = crc_shift_iscsi_sse(crc0, klen * 2 + 8);
        let vc1 = crc_shift_iscsi_sse(crc1, klen + 8);
        let mut vc = _mm_extract_epi64(_mm_xor_si128(vc0, vc1), 0) as u64;

        // Reduce 128 bits to 32 bits, and multiply by x^32
        let x0_low = _mm_extract_epi64(x0, 0) as u64;
        let x0_high = _mm_extract_epi64(x0, 1) as u64;
        let vec_crc = _mm_crc32_u64(_mm_crc32_u64(0, x0_low), x0_high);
        vc ^= _mm_extract_epi64(crc_shift_iscsi_sse(vec_crc as u32, klen * 3 + 8), 0) as u64;

        // Final 8 bytes
        buf = buf.add(klen * 2);
        crc0 = crc2;
        crc0 = _mm_crc32_u64(crc0.into(), *(buf as *const u64) ^ vc) as u32;
        buf = buf.add(8);
        len -= 8;
    }

    // Process remaining 8-byte chunks using hardware CRC32C
    while len >= 8 {
        crc0 = _mm_crc32_u64(crc0.into(), *(buf as *const u64)) as u32;
        buf = buf.add(8);
        len -= 8;
    }

    // Process remaining bytes using hardware CRC32C
    while len > 0 {
        crc0 = _mm_crc32_u8(crc0, *buf);
        buf = buf.add(1);
        len -= 1;
    }

    crc0
}

/// Converted to Rust from the original C code generated by https://github.com/corsix/fast-crc32/
/// using:
///
/// ./generate -i sse -p crc32c -a v4s3x3
#[inline]
#[target_feature(enable = "sse4.2,pclmulqdq")]
pub unsafe fn crc32_iscsi_sse_v4s3x3(mut crc0: u32, mut buf: *const u8, mut len: usize) -> u32 {
    // Align to 8-byte boundary using hardware CRC32C instructions
    while len > 0 && (buf as usize & 7) != 0 {
        crc0 = _mm_crc32_u8(crc0, *buf);
        buf = buf.add(1);
        len -= 1;
    }

    // Handle 8-byte alignment
    if (buf as usize & 8) != 0 && len >= 8 {
        crc0 = mm_crc32_u64(crc0, *(buf as *const u64));
        buf = buf.add(8);
        len -= 8;
    }

    if len >= 144 {
        let blk = (len - 8) / 136;
        let klen = blk * 24;
        let buf2 = buf;
        let mut crc1 = 0u32;
        let mut crc2 = 0u32;

        // First vector chunk - load four 128-bit vectors (64 bytes total)
        let mut x0 = _mm_loadu_si128(buf2 as *const __m128i);
        let mut x1 = _mm_loadu_si128(buf2.add(16) as *const __m128i);
        let mut x2 = _mm_loadu_si128(buf2.add(32) as *const __m128i);
        let mut x3 = _mm_loadu_si128(buf2.add(48) as *const __m128i);

        // iSCSI-specific folding constant (same as AVX-512 version)
        let mut k = _mm_setr_epi32(0x740eef02u32 as i32, 0, 0x9e4addf8u32 as i32, 0);

        // XOR the CRC into the first vector's low 32 bits
        x0 = _mm_xor_si128(_mm_cvtsi32_si128(crc0 as i32), x0);
        crc0 = 0;

        let mut buf2 = buf2.add(64);
        len -= 136;
        buf = buf.add(blk * 64);

        // Main loop - process 144 bytes at a time
        while len >= 144 {
            let mut y0 = clmul_lo_sse(x0, k);
            x0 = clmul_hi_sse(x0, k);
            let mut y1 = clmul_lo_sse(x1, k);
            x1 = clmul_hi_sse(x1, k);
            let mut y2 = clmul_lo_sse(x2, k);
            x2 = clmul_hi_sse(x2, k);
            let mut y3 = clmul_lo_sse(x3, k);
            x3 = clmul_hi_sse(x3, k);

            // XOR operations using separate XOR instructions (no ternary logic in SSE)
            y0 = _mm_xor_si128(y0, _mm_loadu_si128(buf2 as *const __m128i));
            x0 = _mm_xor_si128(x0, y0);
            y1 = _mm_xor_si128(y1, _mm_loadu_si128(buf2.add(16) as *const __m128i));
            x1 = _mm_xor_si128(x1, y1);
            y2 = _mm_xor_si128(y2, _mm_loadu_si128(buf2.add(32) as *const __m128i));
            x2 = _mm_xor_si128(x2, y2);
            y3 = _mm_xor_si128(y3, _mm_loadu_si128(buf2.add(48) as *const __m128i));
            x3 = _mm_xor_si128(x3, y3);

            // Process scalar data in parallel using hardware CRC32C
            crc0 = mm_crc32_u64(crc0, *(buf as *const u64));
            crc1 = mm_crc32_u64(crc1, *(buf.add(klen) as *const u64));
            crc2 = mm_crc32_u64(crc2, *(buf.add(klen * 2) as *const u64));
            crc0 = mm_crc32_u64(crc0, *(buf.add(8) as *const u64));
            crc1 = mm_crc32_u64(crc1, *(buf.add(klen + 8) as *const u64));
            crc2 = mm_crc32_u64(crc2, *(buf.add(klen * 2 + 8) as *const u64));
            crc0 = mm_crc32_u64(crc0, *(buf.add(16) as *const u64));
            crc1 = mm_crc32_u64(crc1, *(buf.add(klen + 16) as *const u64));
            crc2 = mm_crc32_u64(crc2, *(buf.add(klen * 2 + 16) as *const u64));

            buf = buf.add(24);
            buf2 = buf2.add(64);
            len -= 136;
        }

        // Reduce x0 ... x3 to just x0 using iSCSI-specific constants
        k = _mm_setr_epi32(0xf20c0dfeu32 as i32, 0, 0x493c7d27u32 as i32, 0);

        let mut y0 = clmul_lo_sse(x0, k);
        x0 = clmul_hi_sse(x0, k);
        let mut y2 = clmul_lo_sse(x2, k);
        x2 = clmul_hi_sse(x2, k);

        y0 = _mm_xor_si128(y0, x1);
        x0 = _mm_xor_si128(x0, y0);
        y2 = _mm_xor_si128(y2, x3);
        x2 = _mm_xor_si128(x2, y2);

        k = _mm_setr_epi32(0x3da6d0cbu32 as i32, 0, 0xba4fc28eu32 as i32, 0);

        y0 = clmul_lo_sse(x0, k);
        x0 = clmul_hi_sse(x0, k);
        y0 = _mm_xor_si128(y0, x2);
        x0 = _mm_xor_si128(x0, y0);

        // Final scalar chunk
        crc0 = mm_crc32_u64(crc0, *(buf as *const u64));
        crc1 = mm_crc32_u64(crc1, *(buf.add(klen) as *const u64));
        crc2 = mm_crc32_u64(crc2, *(buf.add(klen * 2) as *const u64));
        crc0 = mm_crc32_u64(crc0, *(buf.add(8) as *const u64));
        crc1 = mm_crc32_u64(crc1, *(buf.add(klen + 8) as *const u64));
        crc2 = mm_crc32_u64(crc2, *(buf.add(klen * 2 + 8) as *const u64));
        crc0 = mm_crc32_u64(crc0, *(buf.add(16) as *const u64));
        crc1 = mm_crc32_u64(crc1, *(buf.add(klen + 16) as *const u64));
        crc2 = mm_crc32_u64(crc2, *(buf.add(klen * 2 + 16) as *const u64));
        buf = buf.add(24);

        let vc0 = crc_shift_iscsi_sse(crc0, klen * 2 + 8);
        let vc1 = crc_shift_iscsi_sse(crc1, klen + 8);
        let mut vc = mm_extract_epi64(_mm_xor_si128(vc0, vc1), 0);

        // Reduce 128 bits to 32 bits, and multiply by x^32
        // Extract the two 64-bit parts of x0 and combine them
        let x0_low = mm_extract_epi64(x0, 0);
        let x0_high = mm_extract_epi64(x0, 1);
        let x0_combined = mm_extract_epi64(
            crc_shift_iscsi_sse(mm_crc32_u64(mm_crc32_u64(0, x0_low), x0_high), klen * 3 + 8),
            0,
        );
        vc ^= x0_combined;

        // Final 8 bytes
        buf = buf.add(klen * 2);
        crc0 = crc2;
        crc0 = mm_crc32_u64(crc0, *(buf as *const u64) ^ vc);
        buf = buf.add(8);
        len -= 8;
    }

    // Process remaining 8-byte chunks using hardware CRC32C
    while len >= 8 {
        crc0 = mm_crc32_u64(crc0, *(buf as *const u64));
        buf = buf.add(8);
        len -= 8;
    }

    // Process remaining bytes using hardware CRC32C
    while len > 0 {
        crc0 = _mm_crc32_u8(crc0, *buf);
        buf = buf.add(1);
        len -= 1;
    }

    crc0
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::test::consts::TEST_CHECK_STRING;
    use crc::{Crc, Table};
    use rand::{rng, Rng};

    const RUST_CRC32_ISCSI: Crc<u32, Table<16>> = Crc::<u32, Table<16>>::new(&crc::CRC_32_ISCSI);

    #[test]
    fn test_crc32_iscsi_check() {
        assert_eq!(
            crc32_iscsi(0xffffffff, TEST_CHECK_STRING) ^ 0xffffffff,
            0xe3069283
        );
    }

    #[test]
    fn test_crc32_iscsi_small_all_lengths() {
        for len in 1..=255 {
            test_crc32_iscsi_random(len);
        }
    }

    #[test]
    fn test_crc32_iscsi_medium_lengths() {
        // Test each length from 256 to 1024, which should fold and include handling remainders
        for len in 256..=1024 {
            test_crc32_iscsi_random(len);
        }
    }

    #[test]
    fn test_crc32_iscsi_large_lengths() {
        // Test 1 MiB just before, at, and just after the folding boundaries
        for len in 1048575..1048577 {
            test_crc32_iscsi_random(len);
        }
    }

    //#[rustversion::since(1.89)]
    #[cfg(feature = "vpclmulqdq")]
    fn test_crc32_iscsi_random(len: usize) {
        let mut data = vec![0u8; len];
        rng().fill(&mut data[..]);

        let checksum = RUST_CRC32_ISCSI.checksum(&data);

        assert_eq!(crc32_iscsi(0xffffffff, &data) ^ 0xffffffff, checksum);

        unsafe {
            if is_x86_feature_detected!("vpclmulqdq")
                && is_x86_feature_detected!("avx512vl")
                && is_x86_feature_detected!("avx512f")
            {
                assert_eq!(
                    crc32_iscsi_avx512_vpclmulqdq_v3x2(0xffffffff, data.as_ptr(), data.len())
                        ^ 0xffffffff,
                    checksum
                );
            }

            if is_x86_feature_detected!("avx512vl")
                && is_x86_feature_detected!("avx512f")
                && is_x86_feature_detected!("pclmulqdq")
            {
                assert_eq!(
                    crc32_iscsi_avx512_v4s3x3(0xffffffff, data.as_ptr(), data.len()) ^ 0xffffffff,
                    checksum
                );
            }

            assert_eq!(
                crc32_iscsi_sse_v4s3x3(0xffffffff, data.as_ptr(), data.len()) ^ 0xffffffff,
                checksum
            );
        }
    }

    //#[rustversion::before(1.89)]
    #[cfg(not(feature = "vpclmulqdq"))]
    fn test_crc32_iscsi_random(len: usize) {
        let mut data = vec![0u8; len];
        rng().fill(&mut data[..]);

        let checksum = RUST_CRC32_ISCSI.checksum(&data);

        assert_eq!(crc32_iscsi(0xffffffff, &data) ^ 0xffffffff, checksum);

        unsafe {
            assert_eq!(
                crc32_iscsi_sse_v4s3x3(0xffffffff, data.as_ptr(), data.len()) ^ 0xffffffff,
                checksum
            );
        }
    }
}
