diff --git a/ggml-quants.c b/ggml-quants.c
index 3319d2ccf1812..d32e41ece649c 100644
--- a/ggml-quants.c
+++ b/ggml-quants.c
@@ -53,6 +53,10 @@
 
 #define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)
 
+#if defined(__GNUC__) && __GNUC__ < 8 && !defined(__clang__)
+#define _mm256_set_m128i(a, b) _mm256_permute2f128_si256(_mm256_castsi128_si256(a), _mm256_castsi128_si256(b), 2)
+#endif
+
 #if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__)
 // multiply int8_t, add results pairwise twice
 static inline __m128i mul_sum_i8_pairs(const __m128i x, const __m128i y) {