/******************************************************************************\
* Copyright (c) 2016, Robert van Engelen, Genivia Inc. All rights reserved.    *
*                                                                              *
* Redistribution and use in source and binary forms, with or without           *
* modification, are permitted provided that the following conditions are met:  *
*                                                                              *
*   (1) Redistributions of source code must retain the above copyright notice, *
*       this list of conditions and the following disclaimer.                  *
*                                                                              *
*   (2) Redistributions in binary form must reproduce the above copyright      *
*       notice, this list of conditions and the following disclaimer in the    *
*       documentation and/or other materials provided with the distribution.   *
*                                                                              *
*   (3) The name of the author may not be used to endorse or promote products  *
*       derived from this software without specific prior written permission.  *
*                                                                              *
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED *
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF         *
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO   *
* EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,       *
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, *
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;  *
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,     *
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR      *
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF       *
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.                                   *
\******************************************************************************/

/**
@file      matcher_avx2.cpp
@brief     RE/flex matcher engine
@author    Robert van Engelen - engelen@genivia.com
@copyright (c) 2016-2024, Robert van Engelen, Genivia Inc. All rights reserved.
@copyright (c) BSD-3 License - see LICENSE.txt
*/

#if defined(HAVE_AVX2) || defined(HAVE_AVX512BW)
# if !defined(__AVX2__) && !defined(__AVX512BW__)
#  error simd_avx2.cpp must be compiled with -mavx2 or /arch:avx2.
# endif
#endif

#include <reflex/matcher.h>

namespace reflex {

#if defined(HAVE_AVX2) || defined(HAVE_AVX512BW)

#define SIMD_INIT_ADV_PAT_PIN_CASE_AVX2(PIN) \
  switch (pat_->min_) \
  { \
    case 0: \
    case 1: \
      adv_ = &Matcher::simd_advance_pattern_pin##PIN##_one_avx2; \
      break; \
    case 2: \
    case 3: \
      adv_ = &Matcher::simd_advance_pattern_pin##PIN##_pma_avx2; \
      break; \
    case 4: \
      adv_ = &Matcher::simd_advance_pattern_pin##PIN##_pmh_avx2<4>; \
      break; \
    case 5: \
      adv_ = &Matcher::simd_advance_pattern_pin##PIN##_pmh_avx2<5>; \
      break; \
    case 6: \
      adv_ = &Matcher::simd_advance_pattern_pin##PIN##_pmh_avx2<6>; \
      break; \
    case 7: \
      adv_ = &Matcher::simd_advance_pattern_pin##PIN##_pmh_avx2<7>; \
      break; \
    case 8: \
      adv_ = &Matcher::simd_advance_pattern_pin##PIN##_pmh_avx2<8>; \
      break; \
  }

// AVX2 runtime optimized function callback overrides
void Matcher::simd_init_advance_avx2()
{
  if (pat_->len_ == 0)
  {
    switch (pat_->pin_)
    {
      case 1:
        switch (pat_->min_)
        {
          case 2:
          case 3:
            adv_ = &Matcher::simd_advance_pattern_pin1_pma_avx2;
            break;
          case 4:
            adv_ = &Matcher::simd_advance_pattern_pin1_pmh_avx2<4>;
            break;
          case 5:
            adv_ = &Matcher::simd_advance_pattern_pin1_pmh_avx2<5>;
            break;
          case 6:
            adv_ = &Matcher::simd_advance_pattern_pin1_pmh_avx2<6>;
            break;
          case 7:
            adv_ = &Matcher::simd_advance_pattern_pin1_pmh_avx2<7>;
            break;
          case 8:
            adv_ = &Matcher::simd_advance_pattern_pin1_pmh_avx2<8>;
            break;
        }
        break;
      case 2:
        SIMD_INIT_ADV_PAT_PIN_CASE_AVX2(2);
        break;
      case 3:
        SIMD_INIT_ADV_PAT_PIN_CASE_AVX2(3);
        break;
      case 4:
        SIMD_INIT_ADV_PAT_PIN_CASE_AVX2(4);
        break;
      case 5:
        SIMD_INIT_ADV_PAT_PIN_CASE_AVX2(5);
        break;
      case 6:
        SIMD_INIT_ADV_PAT_PIN_CASE_AVX2(6);
        break;
      case 7:
        SIMD_INIT_ADV_PAT_PIN_CASE_AVX2(7);
        break;
      case 8:
        SIMD_INIT_ADV_PAT_PIN_CASE_AVX2(8);
        break;
      case 16:
        SIMD_INIT_ADV_PAT_PIN_CASE_AVX2(16);
        break;
#ifdef WITH_BITAP_AVX2 // use in case vectorized bitap (hashed) is faster than serial version (typically not!!)
      default:
        switch (pat_->min_)
        {
          case 4:
            adv_ = &Matcher::simd_advance_pattern_min4_avx2<4>;
            break;
          case 5:
            adv_ = &Matcher::simd_advance_pattern_min4_avx2<5>;
            break;
          case 6:
            adv_ = &Matcher::simd_advance_pattern_min4_avx2<6>;
            break;
          case 7:
            adv_ = &Matcher::simd_advance_pattern_min4_avx2<7>;
            break;
          case 8:
            adv_ = &Matcher::simd_advance_pattern_min4_avx2<8>;
            break;
        }
#endif
    }
  }
  else if (pat_->len_ == 1)
  {
    // no specialization
  }
  else if (pat_->len_ == 2)
  {
    if (pat_->min_ == 0)
      adv_ = &Matcher::simd_advance_chars_avx2<2>;
    else if (pat_->min_ < 4)
      adv_ = &Matcher::simd_advance_chars_pma_avx2<2>;
    else
      adv_ = &Matcher::simd_advance_chars_pmh_avx2<2>;
  }
  else if (pat_->len_ == 3)
  {
    if (pat_->min_ == 0)
      adv_ = &Matcher::simd_advance_chars_avx2<3>;
    else if (pat_->min_ < 4)
      adv_ = &Matcher::simd_advance_chars_pma_avx2<3>;
    else
      adv_ = &Matcher::simd_advance_chars_pmh_avx2<3>;
  }
  else if (pat_->bmd_ == 0)
  {
    if (pat_->min_ >= 4)
      adv_ = &Matcher::simd_advance_string_pmh_avx2;
    else if (pat_->min_ > 0)
      adv_ = &Matcher::simd_advance_string_pma_avx2;
    else
      adv_ = &Matcher::simd_advance_string_avx2;
  }
}

// My homegrown "needle search" method when needle pin=1
bool Matcher::simd_advance_pattern_pin1_pma_avx2(size_t loc)
{
  const char *chr = pat_->chr_;
  size_t min = pat_->min_;
  uint16_t lcp = pat_->lcp_;
  uint16_t lcs = pat_->lcs_;
  __m256i vlcp = _mm256_set1_epi8(chr[0]);
  __m256i vlcs = _mm256_set1_epi8(chr[1]);
  while (true)
  {
    const char *s = buf_ + loc + lcp;
    const char *e = buf_ + end_ + lcp - min + 1;
    while (s <= e - 32)
    {
      __m256i vstrlcp = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s));
      __m256i vstrlcs = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s + lcs - lcp));
      __m256i veqlcp = _mm256_cmpeq_epi8(vlcp, vstrlcp);
      __m256i veqlcs = _mm256_cmpeq_epi8(vlcs, vstrlcs);
      uint32_t mask = _mm256_movemask_epi8(_mm256_and_si256(veqlcp, veqlcs));
      while (mask != 0)
      {
        uint32_t offset = ctz(mask);
        loc = s - lcp + offset - buf_;
        set_current(loc);
        if (loc + 4 > end_ || pat_->predict_match(&buf_[loc]))
          return true;
        mask &= mask - 1;
      }
      s += 32;
    }
    loc = s - lcp - buf_;
    set_current_and_peek_more(loc - 1);
    loc = cur_ + 1;
    if (loc + min > end_)
      return false;
    if (loc + min + 31 > end_)
      break;
  }
  return advance_pattern_pin1_pma(loc);
}

// My homegrown "needle search" method when needle pin=1
template <uint8_t MIN>
bool Matcher::simd_advance_pattern_pin1_pmh_avx2(size_t loc)
{
  const char *chr = pat_->chr_;
  uint16_t lcp = pat_->lcp_;
  uint16_t lcs = pat_->lcs_;
  __m256i vlcp = _mm256_set1_epi8(chr[0]);
  __m256i vlcs = _mm256_set1_epi8(chr[1]);
  while (true)
  {
    const char *s = buf_ + loc + lcp;
    const char *e = buf_ + end_ + lcp - MIN + 1;
    while (s <= e - 32)
    {
      __m256i vstrlcp = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s));
      __m256i vstrlcs = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s + lcs - lcp));
      __m256i veqlcp = _mm256_cmpeq_epi8(vlcp, vstrlcp);
      __m256i veqlcs = _mm256_cmpeq_epi8(vlcs, vstrlcs);
      uint32_t mask = _mm256_movemask_epi8(_mm256_and_si256(veqlcp, veqlcs));
      while (mask != 0)
      {
        uint32_t offset = ctz(mask);
        loc = s - lcp + offset - buf_;
        set_current(loc);
        if (pat_->predict_match(&buf_[loc], MIN))
          return true;
        mask &= mask - 1;
      }
      s += 32;
    }
    loc = s - lcp - buf_;
    set_current_and_peek_more(loc - 1);
    loc = cur_ + 1;
    if (loc + MIN > end_)
      return false;
    if (loc + MIN + 31 > end_)
      break;
  }
  return advance_pattern_pin1_pmh<MIN>(loc);
}

// My homegrown "needle search" methods
#define ADV_PAT_PIN_ONE(N, INIT, COMP) \
bool Matcher::simd_advance_pattern_pin##N##_one_avx2(size_t loc) \
{ \
  const char *chr = pat_->chr_; \
  INIT \
  while (true) \
  { \
    const char *s = buf_ + loc; \
    const char *e = buf_ + end_; \
    while (s <= e - 32) \
    { \
      __m256i vstr = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s)); \
      __m256i veq = _mm256_cmpeq_epi8(v0, vstr); \
      COMP \
      uint32_t mask = _mm256_movemask_epi8(veq); \
      while (mask != 0) \
      { \
        uint32_t offset = ctz(mask); \
        loc = s + offset - buf_; \
        if (loc + 4 > end_ || pat_->predict_match(&buf_[loc])) \
        { \
          set_current(loc); \
          return true; \
        } \
        mask &= mask - 1; \
      } \
      s += 32; \
    } \
    loc = s - buf_; \
    set_current_and_peek_more(loc - 1); \
    loc = cur_ + 1; \
    if (loc + 1 > end_) \
      return false; \
    if (loc + 32 > end_) \
      break; \
  } \
  return advance_pattern_pma(loc); \
}

ADV_PAT_PIN_ONE(2, \
    __m256i v0 = _mm256_set1_epi8(chr[0]); \
    __m256i v1 = _mm256_set1_epi8(chr[1]); \
  , \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v1, vstr)); \
  )

ADV_PAT_PIN_ONE(3, \
    __m256i v0 = _mm256_set1_epi8(chr[0]); \
    __m256i v1 = _mm256_set1_epi8(chr[1]); \
    __m256i v2 = _mm256_set1_epi8(chr[2]); \
  , \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v1, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v2, vstr)); \
  )

ADV_PAT_PIN_ONE(4, \
    __m256i v0 = _mm256_set1_epi8(chr[0]); \
    __m256i v1 = _mm256_set1_epi8(chr[1]); \
    __m256i v2 = _mm256_set1_epi8(chr[2]); \
    __m256i v3 = _mm256_set1_epi8(chr[3]); \
  , \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v1, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v2, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v3, vstr)); \
  )

ADV_PAT_PIN_ONE(5, \
    __m256i v0 = _mm256_set1_epi8(chr[0]); \
    __m256i v1 = _mm256_set1_epi8(chr[1]); \
    __m256i v2 = _mm256_set1_epi8(chr[2]); \
    __m256i v3 = _mm256_set1_epi8(chr[3]); \
    __m256i v4 = _mm256_set1_epi8(chr[4]); \
  , \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v1, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v2, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v3, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v4, vstr)); \
  )

ADV_PAT_PIN_ONE(6, \
    __m256i v0 = _mm256_set1_epi8(chr[0]); \
    __m256i v1 = _mm256_set1_epi8(chr[1]); \
    __m256i v2 = _mm256_set1_epi8(chr[2]); \
    __m256i v3 = _mm256_set1_epi8(chr[3]); \
    __m256i v4 = _mm256_set1_epi8(chr[4]); \
    __m256i v5 = _mm256_set1_epi8(chr[5]); \
  , \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v1, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v2, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v3, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v4, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v5, vstr)); \
  )

ADV_PAT_PIN_ONE(7, \
    __m256i v0 = _mm256_set1_epi8(chr[0]); \
    __m256i v1 = _mm256_set1_epi8(chr[1]); \
    __m256i v2 = _mm256_set1_epi8(chr[2]); \
    __m256i v3 = _mm256_set1_epi8(chr[3]); \
    __m256i v4 = _mm256_set1_epi8(chr[4]); \
    __m256i v5 = _mm256_set1_epi8(chr[5]); \
    __m256i v6 = _mm256_set1_epi8(chr[6]); \
  , \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v1, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v2, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v3, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v4, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v5, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v6, vstr)); \
  )

ADV_PAT_PIN_ONE(8, \
    __m256i v0 = _mm256_set1_epi8(chr[0]); \
    __m256i v1 = _mm256_set1_epi8(chr[1]); \
    __m256i v2 = _mm256_set1_epi8(chr[2]); \
    __m256i v3 = _mm256_set1_epi8(chr[3]); \
    __m256i v4 = _mm256_set1_epi8(chr[4]); \
    __m256i v5 = _mm256_set1_epi8(chr[5]); \
    __m256i v6 = _mm256_set1_epi8(chr[6]); \
    __m256i v7 = _mm256_set1_epi8(chr[7]); \
  , \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v1, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v2, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v3, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v4, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v5, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v6, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v7, vstr)); \
  )

ADV_PAT_PIN_ONE(16, \
    __m256i v0 = _mm256_set1_epi8(chr[0]); \
    __m256i v1 = _mm256_set1_epi8(chr[1]); \
    __m256i v2 = _mm256_set1_epi8(chr[2]); \
    __m256i v3 = _mm256_set1_epi8(chr[3]); \
    __m256i v4 = _mm256_set1_epi8(chr[4]); \
    __m256i v5 = _mm256_set1_epi8(chr[5]); \
    __m256i v6 = _mm256_set1_epi8(chr[6]); \
    __m256i v7 = _mm256_set1_epi8(chr[7]); \
    __m256i v8 = _mm256_set1_epi8(chr[8]); \
    __m256i v9 = _mm256_set1_epi8(chr[9]); \
    __m256i va = _mm256_set1_epi8(chr[10]); \
    __m256i vb = _mm256_set1_epi8(chr[11]); \
    __m256i vc = _mm256_set1_epi8(chr[12]); \
    __m256i vd = _mm256_set1_epi8(chr[13]); \
    __m256i ve = _mm256_set1_epi8(chr[14]); \
    __m256i vf = _mm256_set1_epi8(chr[15]); \
  , \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v1, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v2, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v3, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v4, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v5, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v6, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v7, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v8, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(v9, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(va, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(vb, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(vc, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(vd, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(ve, vstr)); \
    veq = _mm256_or_si256(veq, _mm256_cmpeq_epi8(vf, vstr)); \
  )

// My homegrown "needle search" methods
#define ADV_PAT_PIN(N, INIT, COMP) \
bool Matcher::simd_advance_pattern_pin##N##_pma_avx2(size_t loc) \
{ \
  const char *chr = pat_->chr_; \
  size_t min = pat_->min_; \
  uint16_t lcp = pat_->lcp_; \
  uint16_t lcs = pat_->lcs_; \
  INIT \
  while (true) \
  { \
    const char *s = buf_ + loc + lcp; \
    const char *e = buf_ + end_ + lcp - min + 1; \
    while (s <= e - 32) \
    { \
      __m256i vstrlcp = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s)); \
      __m256i vstrlcs = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s + lcs - lcp)); \
      __m256i veqlcp = _mm256_cmpeq_epi8(vlcp0, vstrlcp); \
      __m256i veqlcs = _mm256_cmpeq_epi8(vlcs0, vstrlcs); \
      COMP \
      uint32_t mask = _mm256_movemask_epi8(_mm256_and_si256(veqlcp, veqlcs)); \
      while (mask != 0) \
      { \
        uint32_t offset = ctz(mask); \
        loc = s - lcp + offset - buf_; \
        if (loc + 4 > end_ || pat_->predict_match(&buf_[loc])) \
        { \
          set_current(loc); \
          return true; \
        } \
        mask &= mask - 1; \
      } \
      s += 32; \
    } \
    loc = s - lcp - buf_; \
    set_current_and_peek_more(loc - 1); \
    loc = cur_ + 1; \
    if (loc + min > end_) \
      return false; \
    if (loc + min + 31 > end_) \
      break; \
  } \
  return advance_pattern_pma(loc); \
} \
\
template <uint8_t MIN> \
bool Matcher::simd_advance_pattern_pin##N##_pmh_avx2(size_t loc) \
{ \
  const char *chr = pat_->chr_; \
  uint16_t lcp = pat_->lcp_; \
  uint16_t lcs = pat_->lcs_; \
  INIT \
  while (true) \
  { \
    const char *s = buf_ + loc + lcp; \
    const char *e = buf_ + end_ + lcp - MIN + 1; \
    while (s <= e - 32) \
    { \
      __m256i vstrlcp = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s)); \
      __m256i vstrlcs = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s + lcs - lcp)); \
      __m256i veqlcp = _mm256_cmpeq_epi8(vlcp0, vstrlcp); \
      __m256i veqlcs = _mm256_cmpeq_epi8(vlcs0, vstrlcs); \
      COMP \
      uint32_t mask = _mm256_movemask_epi8(_mm256_and_si256(veqlcp, veqlcs)); \
      while (mask != 0) \
      { \
        uint32_t offset = ctz(mask); \
        loc = s - lcp + offset - buf_; \
        if (pat_->predict_match(&buf_[loc], MIN)) \
        { \
          set_current(loc); \
          return true; \
        } \
        mask &= mask - 1; \
      } \
      s += 32; \
    } \
    loc = s - lcp - buf_; \
    set_current_and_peek_more(loc - 1); \
    loc = cur_ + 1; \
    if (loc + MIN > end_) \
      return false; \
    if (loc + MIN + 31 > end_) \
      break; \
  } \
  return advance_pattern_min4<MIN>(loc); \
}

ADV_PAT_PIN(2, \
    __m256i vlcp0 = _mm256_set1_epi8(chr[0]); \
    __m256i vlcp1 = _mm256_set1_epi8(chr[1]); \
    __m256i vlcs0 = _mm256_set1_epi8(chr[2]); \
    __m256i vlcs1 = _mm256_set1_epi8(chr[3]); \
  , \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp1, vstrlcp)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs1, vstrlcs)); \
  )

ADV_PAT_PIN(3, \
    __m256i vlcp0 = _mm256_set1_epi8(chr[0]); \
    __m256i vlcp1 = _mm256_set1_epi8(chr[1]); \
    __m256i vlcp2 = _mm256_set1_epi8(chr[2]); \
    __m256i vlcs0 = _mm256_set1_epi8(chr[3]); \
    __m256i vlcs1 = _mm256_set1_epi8(chr[4]); \
    __m256i vlcs2 = _mm256_set1_epi8(chr[5]); \
  , \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp1, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp2, vstrlcp)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs1, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs2, vstrlcs)); \
  )

ADV_PAT_PIN(4, \
    __m256i vlcp0 = _mm256_set1_epi8(chr[0]); \
    __m256i vlcp1 = _mm256_set1_epi8(chr[1]); \
    __m256i vlcp2 = _mm256_set1_epi8(chr[2]); \
    __m256i vlcp3 = _mm256_set1_epi8(chr[3]); \
    __m256i vlcs0 = _mm256_set1_epi8(chr[4]); \
    __m256i vlcs1 = _mm256_set1_epi8(chr[5]); \
    __m256i vlcs2 = _mm256_set1_epi8(chr[6]); \
    __m256i vlcs3 = _mm256_set1_epi8(chr[7]); \
  , \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp1, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp2, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp3, vstrlcp)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs1, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs2, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs3, vstrlcs)); \
  )

ADV_PAT_PIN(5, \
    __m256i vlcp0 = _mm256_set1_epi8(chr[0]); \
    __m256i vlcp1 = _mm256_set1_epi8(chr[1]); \
    __m256i vlcp2 = _mm256_set1_epi8(chr[2]); \
    __m256i vlcp3 = _mm256_set1_epi8(chr[3]); \
    __m256i vlcp4 = _mm256_set1_epi8(chr[4]); \
    __m256i vlcs0 = _mm256_set1_epi8(chr[5]); \
    __m256i vlcs1 = _mm256_set1_epi8(chr[6]); \
    __m256i vlcs2 = _mm256_set1_epi8(chr[7]); \
    __m256i vlcs3 = _mm256_set1_epi8(chr[8]); \
    __m256i vlcs4 = _mm256_set1_epi8(chr[9]); \
  , \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp1, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp2, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp3, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp4, vstrlcp)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs1, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs2, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs3, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs4, vstrlcs)); \
  )

ADV_PAT_PIN(6, \
    __m256i vlcp0 = _mm256_set1_epi8(chr[0]); \
    __m256i vlcp1 = _mm256_set1_epi8(chr[1]); \
    __m256i vlcp2 = _mm256_set1_epi8(chr[2]); \
    __m256i vlcp3 = _mm256_set1_epi8(chr[3]); \
    __m256i vlcp4 = _mm256_set1_epi8(chr[4]); \
    __m256i vlcp5 = _mm256_set1_epi8(chr[5]); \
    __m256i vlcs0 = _mm256_set1_epi8(chr[6]); \
    __m256i vlcs1 = _mm256_set1_epi8(chr[7]); \
    __m256i vlcs2 = _mm256_set1_epi8(chr[8]); \
    __m256i vlcs3 = _mm256_set1_epi8(chr[9]); \
    __m256i vlcs4 = _mm256_set1_epi8(chr[10]); \
    __m256i vlcs5 = _mm256_set1_epi8(chr[11]); \
  , \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp1, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp2, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp3, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp4, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp5, vstrlcp)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs1, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs2, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs3, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs4, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs5, vstrlcs)); \
  )

ADV_PAT_PIN(7, \
    __m256i vlcp0 = _mm256_set1_epi8(chr[0]); \
    __m256i vlcp1 = _mm256_set1_epi8(chr[1]); \
    __m256i vlcp2 = _mm256_set1_epi8(chr[2]); \
    __m256i vlcp3 = _mm256_set1_epi8(chr[3]); \
    __m256i vlcp4 = _mm256_set1_epi8(chr[4]); \
    __m256i vlcp5 = _mm256_set1_epi8(chr[5]); \
    __m256i vlcp6 = _mm256_set1_epi8(chr[6]); \
    __m256i vlcs0 = _mm256_set1_epi8(chr[7]); \
    __m256i vlcs1 = _mm256_set1_epi8(chr[8]); \
    __m256i vlcs2 = _mm256_set1_epi8(chr[9]); \
    __m256i vlcs3 = _mm256_set1_epi8(chr[10]); \
    __m256i vlcs4 = _mm256_set1_epi8(chr[11]); \
    __m256i vlcs5 = _mm256_set1_epi8(chr[12]); \
    __m256i vlcs6 = _mm256_set1_epi8(chr[13]); \
  , \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp1, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp2, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp3, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp4, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp5, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp6, vstrlcp)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs1, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs2, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs3, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs4, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs5, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs6, vstrlcs)); \
  )

ADV_PAT_PIN(8, \
    __m256i vlcp0 = _mm256_set1_epi8(chr[0]); \
    __m256i vlcp1 = _mm256_set1_epi8(chr[1]); \
    __m256i vlcp2 = _mm256_set1_epi8(chr[2]); \
    __m256i vlcp3 = _mm256_set1_epi8(chr[3]); \
    __m256i vlcp4 = _mm256_set1_epi8(chr[4]); \
    __m256i vlcp5 = _mm256_set1_epi8(chr[5]); \
    __m256i vlcp6 = _mm256_set1_epi8(chr[6]); \
    __m256i vlcp7 = _mm256_set1_epi8(chr[7]); \
    __m256i vlcs0 = _mm256_set1_epi8(chr[8]); \
    __m256i vlcs1 = _mm256_set1_epi8(chr[9]); \
    __m256i vlcs2 = _mm256_set1_epi8(chr[10]); \
    __m256i vlcs3 = _mm256_set1_epi8(chr[11]); \
    __m256i vlcs4 = _mm256_set1_epi8(chr[12]); \
    __m256i vlcs5 = _mm256_set1_epi8(chr[13]); \
    __m256i vlcs6 = _mm256_set1_epi8(chr[14]); \
    __m256i vlcs7 = _mm256_set1_epi8(chr[15]); \
  , \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp1, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp2, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp3, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp4, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp5, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp6, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp7, vstrlcp)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs1, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs2, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs3, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs4, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs5, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs6, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs7, vstrlcs)); \
  )

ADV_PAT_PIN(16, \
    __m256i vlcp0 = _mm256_set1_epi8(chr[0]); \
    __m256i vlcp1 = _mm256_set1_epi8(chr[1]); \
    __m256i vlcp2 = _mm256_set1_epi8(chr[2]); \
    __m256i vlcp3 = _mm256_set1_epi8(chr[3]); \
    __m256i vlcp4 = _mm256_set1_epi8(chr[4]); \
    __m256i vlcp5 = _mm256_set1_epi8(chr[5]); \
    __m256i vlcp6 = _mm256_set1_epi8(chr[6]); \
    __m256i vlcp7 = _mm256_set1_epi8(chr[7]); \
    __m256i vlcp8 = _mm256_set1_epi8(chr[8]); \
    __m256i vlcp9 = _mm256_set1_epi8(chr[9]); \
    __m256i vlcpa = _mm256_set1_epi8(chr[10]); \
    __m256i vlcpb = _mm256_set1_epi8(chr[11]); \
    __m256i vlcpc = _mm256_set1_epi8(chr[12]); \
    __m256i vlcpd = _mm256_set1_epi8(chr[13]); \
    __m256i vlcpe = _mm256_set1_epi8(chr[14]); \
    __m256i vlcpf = _mm256_set1_epi8(chr[15]); \
    __m256i vlcs0 = _mm256_set1_epi8(chr[16]); \
    __m256i vlcs1 = _mm256_set1_epi8(chr[17]); \
    __m256i vlcs2 = _mm256_set1_epi8(chr[18]); \
    __m256i vlcs3 = _mm256_set1_epi8(chr[19]); \
    __m256i vlcs4 = _mm256_set1_epi8(chr[20]); \
    __m256i vlcs5 = _mm256_set1_epi8(chr[21]); \
    __m256i vlcs6 = _mm256_set1_epi8(chr[22]); \
    __m256i vlcs7 = _mm256_set1_epi8(chr[23]); \
    __m256i vlcs8 = _mm256_set1_epi8(chr[24]); \
    __m256i vlcs9 = _mm256_set1_epi8(chr[25]); \
    __m256i vlcsa = _mm256_set1_epi8(chr[26]); \
    __m256i vlcsb = _mm256_set1_epi8(chr[27]); \
    __m256i vlcsc = _mm256_set1_epi8(chr[28]); \
    __m256i vlcsd = _mm256_set1_epi8(chr[29]); \
    __m256i vlcse = _mm256_set1_epi8(chr[30]); \
    __m256i vlcsf = _mm256_set1_epi8(chr[31]); \
  , \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp1, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp2, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp3, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp4, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp5, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp6, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp7, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp8, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcp9, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcpa, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcpb, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcpc, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcpd, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcpe, vstrlcp)); \
    veqlcp = _mm256_or_si256(veqlcp, _mm256_cmpeq_epi8(vlcpf, vstrlcp)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs1, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs2, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs3, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs4, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs5, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs6, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs7, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs8, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcs9, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcsa, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcsb, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcsc, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcsd, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcse, vstrlcs)); \
    veqlcs = _mm256_or_si256(veqlcs, _mm256_cmpeq_epi8(vlcsf, vstrlcs)); \
  )

#ifdef WITH_BITAP_AVX2 // use in case vectorized bitap (hashed) is faster than serial version (typically not!!)

/// Minimal 4 byte long patterns (MIN>=4) using bitap hashed pairs with AVX2
template <uint8_t MIN>
bool Matcher::simd_advance_pattern_min4_avx2(size_t loc)
{
  const uint32_t btap = Pattern::Const::BTAP;
  const __m128i vmod = _mm_set1_epi32(btap - 1);
  const __m128i vselect = _mm_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 13, 9, 5, 1);
  const __m128i voffset = _mm_set_epi32(0, btap, 2 * btap, 3 * btap);
  uint32_t state0 = ~0U << (8 - (MIN - 1));
  uint32_t state1 = ~0U << (8 - (MIN - 2));
  uint32_t state2 = ~0U << (8 - (MIN - 3));
  uint32_t state3 = ~0U << (8 - (MIN - 4));
  if (MIN <= 6)
    state3 = state2;
  if (MIN <= 5)
    state2 = state1;
  if (MIN <= 4)
    state1 = state0;
  __m128i vstate = _mm_set_epi32(state0, state1, state2, state3);
  __m128i vc0 = _mm_set1_epi32(buf_[loc++]);
  while (true)
  {
    const char *s = buf_ + loc;
    const char *e = buf_ + end_ - 3;
    while (s < e)
    {
      __m128i vc1 = _mm_cvtepu8_epi32(_mm_loadu_si32(reinterpret_cast<const uint32_t*>(s)));
      vc0 = _mm_alignr_epi8(vc1, vc0, 12);
      // hash
      __m128i vh = _mm_and_si128(_mm_xor_si128(vc0, _mm_slli_epi32(vc1, 6)), vmod);
      // gather bitap hashed bits
      __m128i vb = _mm_i32gather_epi32(reinterpret_cast<const int32_t*>(pat_->vtp_), _mm_or_si128(vh, voffset), 2);
      // shift-or
      vstate = _mm_or_si128(_mm_slli_epi32(vstate, 4), vb);
      // pass last char to the next iteration
      vc0 = vc1;
      // get most significant bit of each byte, check each 2nd byte of the 4x32 bit words
      uint32_t mask = _mm_extract_epi32(_mm_shuffle_epi8(vstate, vselect), 0);
      if ((mask & 0x00000008) == 0 && pat_->predict_match(s - MIN + 0, MIN))
      {
        loc = s - buf_ - MIN + 0;
        set_current(loc);
        return true;
      }
      if ((mask & 0x00000404) == 0 && pat_->predict_match(s - MIN + 1, MIN))
      {
        loc = s - buf_ - MIN + 1;
        set_current(loc);
        return true;
      }
      if ((mask & 0x00020202) == 0 && pat_->predict_match(s - MIN + 2, MIN))
      {
        loc = s - buf_ - MIN + 2;
        set_current(loc);
        return true;
      }
      if ((mask & 0x01010101) == 0 && pat_->predict_match(s - MIN + 3, MIN))
      {
        loc = s - buf_ - MIN + 3;
        set_current(loc);
        return true;
      }
      // butterfly-or:
      //    a       b       c       d     input vec
      //    c       d       a       b     swizzle
      //   a|c     b|d     c|a     d|b    or
      //   b|d     a|c     d|b     c|a    swizzle
      // a|c|b|d b|d|a|c c|a|d|b d|b|c|a  or
      vstate = _mm_or_si128(vstate, _mm_shuffle_epi32(vstate, 0x4e)); // = 01 00 11 10 = 1 0 3 2
      vstate = _mm_or_si128(vstate, _mm_shuffle_epi32(vstate, 0xb1)); // = 10 11 00 01 = 2 3 0 1
      s += 4;
    }
    loc = s - buf_;
    set_current_and_peek_more(loc - 1);
    loc = cur_ + 1;
    if (loc + 3 >= end_)
      return advance_pattern_min4<MIN>(loc - MIN);
  }
}

#endif

/// Few chars
template<uint8_t LEN>
bool Matcher::simd_advance_chars_avx2(size_t loc)
{
  static const uint16_t lcp = 0;
  static const uint16_t lcs = LEN - 1;
  const char *chr = pat_->chr_;
  while (true)
  {
    const char *s = buf_ + loc + lcp;
    const char *e = buf_ + end_ + lcp - LEN + 1;
    __m256i vlcp = _mm256_set1_epi8(chr[lcp]);
    __m256i vlcs = _mm256_set1_epi8(chr[lcs]);
    while (s <= e - 32)
    {
      __m256i vlcpm = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s));
      __m256i vlcsm = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s + lcs - lcp));
      __m256i vlcpeq = _mm256_cmpeq_epi8(vlcp, vlcpm);
      __m256i vlcseq = _mm256_cmpeq_epi8(vlcs, vlcsm);
      uint32_t mask = _mm256_movemask_epi8(_mm256_and_si256(vlcpeq, vlcseq));
      while (mask != 0)
      {
        uint32_t offset = ctz(mask);
        if (LEN == 2 ||
            (LEN == 3 ? s[offset + 1 - lcp] == chr[1] : std::memcmp(s + 1 - lcp + offset, chr + 1, LEN - 2) == 0))
        {
          loc = s - lcp + offset - buf_;
          set_current(loc);
          return true;
        }
        mask &= mask - 1;
      }
      s += 32;
    }
    loc = s - lcp - buf_;
    set_current_and_peek_more(loc - 1);
    loc = cur_ + 1;
    if (loc + LEN > end_)
      return false;
    if (loc + LEN + 31 > end_)
      break;
  }
  return advance_chars<LEN>(loc);
}

/// Few chars followed by 2 to 3 minimal char pattern
template<uint8_t LEN>
bool Matcher::simd_advance_chars_pma_avx2(size_t loc)
{
  static const uint16_t lcp = 0;
  static const uint16_t lcs = LEN - 1;
  const char *chr = pat_->chr_;
  size_t min = pat_->min_;
  while (true)
  {
    const char *s = buf_ + loc + lcp;
    const char *e = buf_ + end_ + lcp - LEN + 1;
    __m256i vlcp = _mm256_set1_epi8(chr[lcp]);
    __m256i vlcs = _mm256_set1_epi8(chr[lcs]);
    while (s <= e - 32)
    {
      __m256i vlcpm = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s));
      __m256i vlcsm = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s + lcs - lcp));
      __m256i vlcpeq = _mm256_cmpeq_epi8(vlcp, vlcpm);
      __m256i vlcseq = _mm256_cmpeq_epi8(vlcs, vlcsm);
      uint32_t mask = _mm256_movemask_epi8(_mm256_and_si256(vlcpeq, vlcseq));
      while (mask != 0)
      {
        uint32_t offset = ctz(mask);
        if (LEN == 2 ||
            (LEN == 3 ? s[offset + 1 - lcp] : std::memcmp(s + 1 - lcp + offset, chr + 1, LEN - 2) == 0))
        {
          loc = s - lcp + offset - buf_;
          if (loc + LEN + 4 > end_ || pat_->predict_match(&buf_[loc + LEN]))
          {
            set_current(loc);
            return true;
          }
        }
        mask &= mask - 1;
      }
      s += 32;
    }
    loc = s - lcp - buf_;
    set_current_and_peek_more(loc - 1);
    loc = cur_ + 1;
    if (loc + LEN + min > end_)
      return false;
    if (loc + LEN + min + 31 > end_)
      break;
  }
  return advance_chars_pma<LEN>(loc);
}

/// Few chars followed by 4 minimal char pattern
template<uint8_t LEN>
bool Matcher::simd_advance_chars_pmh_avx2(size_t loc)
{
  static const uint16_t lcp = 0;
  static const uint16_t lcs = LEN - 1;
  const char *chr = pat_->chr_;
  size_t min = pat_->min_;
  while (true)
  {
    const char *s = buf_ + loc + lcp;
    const char *e = buf_ + end_ + lcp - LEN + 1;
    __m256i vlcp = _mm256_set1_epi8(chr[lcp]);
    __m256i vlcs = _mm256_set1_epi8(chr[lcs]);
    while (s <= e - 32)
    {
      __m256i vlcpm = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s));
      __m256i vlcsm = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s + lcs - lcp));
      __m256i vlcpeq = _mm256_cmpeq_epi8(vlcp, vlcpm);
      __m256i vlcseq = _mm256_cmpeq_epi8(vlcs, vlcsm);
      uint32_t mask = _mm256_movemask_epi8(_mm256_and_si256(vlcpeq, vlcseq));
      while (mask != 0)
      {
        uint32_t offset = ctz(mask);
        if (LEN == 2 ||
            (LEN == 3 ? s[offset + 1 - lcp] == chr[1] : std::memcmp(s + 1 - lcp + offset, chr + 1, LEN - 2) == 0))
        {
          loc = s - lcp + offset - buf_;
          set_current(loc);
          if (loc + LEN + min > end_ || pat_->predict_match(&buf_[loc + LEN], min))
            return true;
        }
        mask &= mask - 1;
      }
      s += 32;
    }
    loc = s - lcp - buf_;
    set_current_and_peek_more(loc - 1);
    loc = cur_ + 1;
    if (loc + LEN + min > end_)
      return false;
    if (loc + LEN + min + 31 > end_)
      break;
  }
  return advance_chars_pmh<LEN>(loc);
}

/// Implements AVX2 string search scheme based on http://0x80.pl/articles/simd-friendly-karp-rabin.html
bool Matcher::simd_advance_string_avx2(size_t loc)
{
  const char *chr = pat_->chr_;
  size_t len = pat_->len_;
  uint16_t lcp = pat_->lcp_;
  uint16_t lcs = pat_->lcs_;
  while (true)
  {
    const char *s = buf_ + loc + lcp;
    const char *e = buf_ + end_ + lcp - len + 1;
    __m256i vlcp = _mm256_set1_epi8(chr[lcp]);
    __m256i vlcs = _mm256_set1_epi8(chr[lcs]);
    while (s <= e - 32)
    {
      __m256i vlcpm = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s));
      __m256i vlcsm = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s + lcs - lcp));
      __m256i vlcpeq = _mm256_cmpeq_epi8(vlcp, vlcpm);
      __m256i vlcseq = _mm256_cmpeq_epi8(vlcs, vlcsm);
      uint32_t mask = _mm256_movemask_epi8(_mm256_and_si256(vlcpeq, vlcseq));
      while (mask != 0)
      {
        uint32_t offset = ctz(mask);
        if (std::memcmp(s - lcp + offset, chr, len) == 0)
        {
          loc = s - lcp + offset - buf_;
          set_current(loc);
          return true;
        }
        mask &= mask - 1;
      }
      s += 32;
    }
    loc = s - lcp - buf_;
    set_current_and_peek_more(loc - 1);
    loc = cur_ + 1;
    if (loc + len > end_)
      return false;
    if (loc + len + 31 > end_)
      break;
  }
  return advance_string(loc);
}

/// Implements AVX2 string search scheme based on http://0x80.pl/articles/simd-friendly-karp-rabin.html
bool Matcher::simd_advance_string_pma_avx2(size_t loc)
{
  const char *chr = pat_->chr_;
  size_t len = pat_->len_;
  size_t min = pat_->min_;
  uint16_t lcp = pat_->lcp_;
  uint16_t lcs = pat_->lcs_;
  while (true)
  {
    const char *s = buf_ + loc + lcp;
    const char *e = buf_ + end_ + lcp - len + 1;
    __m256i vlcp = _mm256_set1_epi8(chr[lcp]);
    __m256i vlcs = _mm256_set1_epi8(chr[lcs]);
    while (s <= e - 32)
    {
      __m256i vlcpm = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s));
      __m256i vlcsm = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s + lcs - lcp));
      __m256i vlcpeq = _mm256_cmpeq_epi8(vlcp, vlcpm);
      __m256i vlcseq = _mm256_cmpeq_epi8(vlcs, vlcsm);
      uint32_t mask = _mm256_movemask_epi8(_mm256_and_si256(vlcpeq, vlcseq));
      while (mask != 0)
      {
        uint32_t offset = ctz(mask);
        if (std::memcmp(s - lcp + offset, chr, len) == 0)
        {
          loc = s - lcp + offset - buf_;
          if (loc + len + 4 > end_ || pat_->predict_match(&buf_[loc + len]))
          {
            set_current(loc);
            return true;
          }
        }
        mask &= mask - 1;
      }
      s += 32;
    }
    loc = s - lcp - buf_;
    set_current_and_peek_more(loc - 1);
    loc = cur_ + 1;
    if (loc + len + min > end_)
      return false;
    if (loc + len + min + 31 > end_)
      break;
  }
  return advance_string_pma(loc);
}

/// Implements AVX2 string search scheme based on http://0x80.pl/articles/simd-friendly-karp-rabin.html
bool Matcher::simd_advance_string_pmh_avx2(size_t loc)
{
  const char *chr = pat_->chr_;
  size_t len = pat_->len_;
  size_t min = pat_->min_;
  uint16_t lcp = pat_->lcp_;
  uint16_t lcs = pat_->lcs_;
  while (true)
  {
    const char *s = buf_ + loc + lcp;
    const char *e = buf_ + end_ + lcp - len + 1;
    __m256i vlcp = _mm256_set1_epi8(chr[lcp]);
    __m256i vlcs = _mm256_set1_epi8(chr[lcs]);
    while (s <= e - 32)
    {
      __m256i vlcpm = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s));
      __m256i vlcsm = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s + lcs - lcp));
      __m256i vlcpeq = _mm256_cmpeq_epi8(vlcp, vlcpm);
      __m256i vlcseq = _mm256_cmpeq_epi8(vlcs, vlcsm);
      uint32_t mask = _mm256_movemask_epi8(_mm256_and_si256(vlcpeq, vlcseq));
      while (mask != 0)
      {
        uint32_t offset = ctz(mask);
        if (std::memcmp(s - lcp + offset, chr, len) == 0)
        {
          loc = s - lcp + offset - buf_;
          set_current(loc);
          if (loc + len + min > end_ || pat_->predict_match(&buf_[loc + len], min))
            return true;
        }
        mask &= mask - 1;
      }
      s += 32;
    }
    loc = s - lcp - buf_;
    set_current_and_peek_more(loc - 1);
    loc = cur_ + 1;
    if (loc + len + min > end_)
      return false;
    if (loc + len + min + 31 > end_)
      break;
  }
  return advance_string_pmh(loc);
}

#else

// appease ranlib "has no symbols"
void matcher_not_compiled_with_avx2() { }

#endif

} // namespace reflex
