// Copyright 2010 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Implements multiword CRC for GCC on AMD64.
//
// Accoding to "Software Optimization Guide for AMD Family 10h Processors"
// http://www.amd.com/us-en/assets/content_type/white_papers_and_tech_docs/40546.pdf
// instead of
// movzbq %al, %rsi
// shrq $8, %rax
// [use %rsi]
// movzbq %al, %rsi
// shrq $8, %rax
// [use %rsi]
// it is better to use 32-bit registers
// (high 32 bits will be cleared on assignment), i.e.
// movzbl %al, %esi
// [use %rsi]
// movzbl %ah, %esi
// shrq $16, %rax
// [use %rsi]
// Makes instructions shorter and removes one shift
// (the latter is not such a big deal as it's execution time
// is nicely masked by [use %rsi] instruction).
//
// Performance difference:
// About 10% degradation on bytes = 8 .. 16
// (clobbering registers that should be saved)
// Break even at 32 bytes.
// 3% improvement starting from 64 bytes.
#include "generic_crc.h"
#if defined(__GNUC__) && CRCUTIL_USE_ASM && HAVE_AMD64
namespace crcutil {
template<> uint64 GenericCrc<uint64, uint64, uint64, 4>::CrcMultiwordGccAmd64(
const void *data, size_t bytes, const uint64 &start) const;
template<> uint64 GenericCrc<uint64, uint64, uint64, 4>::CrcMultiword(
const void *data,
size_t bytes,
const uint64 &start) const {
if (bytes <= 6 * sizeof(Word) - 1) {
const uint8 *src = static_cast<const uint8 *>(data);
uint64 crc = start ^ this->Base().Canonize();
const uint8 *end = src + bytes;
#define PROCESS_ONE_WORD() do { \
Word buf = reinterpret_cast<const Word *>(src)[0]; \
CRC_WORD(this, crc, buf); \
src += sizeof(Word); \
} while (0)
if (bytes >= 1 * sizeof(Word)) {
PROCESS_ONE_WORD();
if (bytes >= 2 * sizeof(Word)) {
PROCESS_ONE_WORD();
if (bytes >= 3 * sizeof(Word)) {
PROCESS_ONE_WORD();
if (bytes >= 4 * sizeof(Word)) {
PROCESS_ONE_WORD();
if (bytes >= 5 * sizeof(Word)) {
PROCESS_ONE_WORD();
}
}
}
}
}
for (; src < end; ++src) {
CRC_BYTE(this, crc, *src);
}
return (crc ^ this->Base().Canonize());
}
return this->CrcMultiwordGccAmd64(data, bytes, start);
}
#define TMP0 "%%rsi"
#define TMP0W "%%esi"
#define BUF0 "%%rax"
#define BUF0L "%%al"
#define BUF0H "%%ah"
#define BUF1 "%%rbx"
#define BUF1L "%%bl"
#define BUF1H "%%bh"
#define BUF2 "%%rcx"
#define BUF2L "%%cl"
#define BUF2H "%%ch"
#define BUF3 "%%rdx"
#define BUF3L "%%dl"
#define BUF3H "%%dh"
#define CRC_WORD_ASM() \
"xorq %[crc0], " BUF0 "\n" \
"movzbq " BUF0L ", " TMP0 "\n" \
"movq (%[table_word], " TMP0 ", 8), %[crc0]\n" \
"movzbl " BUF0H ", " TMP0W "\n" \
"shrq $16, " BUF0 "\n" \
"xorq 1*256*8(%[table_word], " TMP0 ", 8), %[crc0]\n" \
"movzbq " BUF0L ", " TMP0 "\n" \
"xorq 2*256*8(%[table_word], " TMP0 ", 8), %[crc0]\n" \
"movzbl " BUF0H ", " TMP0W "\n" \
"shrq $16, " BUF0 "\n" \
"xorq 3*256*8(%[table_word], " TMP0 ", 8), %[crc0]\n" \
"movzbq " BUF0L ", " TMP0 "\n" \
"xorq 4*256*8(%[table_word], " TMP0 ", 8), %[crc0]\n" \
"movzbl " BUF0H ", " TMP0W "\n" \
"shrq $16, " BUF0 "\n" \
"xorq 5*256*8(%[table_word], " TMP0 ", 8), %[crc0]\n" \
"movzbq " BUF0L ", " TMP0 "\n" \
"xorq 6*256*8(%[table_word], " TMP0 ", 8), %[crc0]\n" \
"movzbl " BUF0H ", " TMP0W "\n" \
"xorq 7*256*8(%[table_word], " TMP0 ", 8), %[crc0]\n"
template<> uint64 GenericCrc<uint64, uint64, uint64, 4>::CrcMultiwordGccAmd64(
const void *data, size_t bytes, const uint64 &start) const {
const uint8 *src = static_cast<const uint8 *>(data);
const uint8 *end = src + bytes;
uint64 crc0 = start ^ this->Base().Canonize();
ALIGN_ON_WORD_BOUNDARY_IF_NEEDED(bytes, this, src, end, crc0, uint64);
if (src >= end) {
return (crc0 ^ this->Base().Canonize());
}
uint64 crc1;
uint64 crc2;
uint64 crc3;
asm(
"sub $2*4*8 - 1, %[end]\n"
"cmpq %[src], %[end]\n"
"jbe 2f\n"
"xorq %[crc1], %[crc1]\n"
"movq (%[src]), " BUF0 "\n"
"movq 1*8(%[src]), " BUF1 "\n"
"movq 2*8(%[src]), " BUF2 "\n"
"movq 3*8(%[src]), " BUF3 "\n"
"movq %[crc1], %[crc2]\n"
"movq %[crc1], %[crc3]\n"
"1:\n"
#if HAVE_SSE && CRCUTIL_PREFETCH_WIDTH > 0
"prefetcht0 " TO_STRING(CRCUTIL_PREFETCH_WIDTH) "(%[src])\n"
#endif // HAVE_SSE
"add $4*8, %[src]\n"
// Set buffer data.
"xorq %[crc0], " BUF0 "\n"
"xorq %[crc1], " BUF1 "\n"
"xorq %[crc2], " BUF2 "\n"
"xorq %[crc3], " BUF3 "\n"
// LOAD crc of byte 0 and shift buffers.
"movzbl " BUF0L ", " TMP0W "\n"
"movq (%[table], " TMP0 ", 8), %[crc0]\n"
"movzbl " BUF1L ", " TMP0W "\n"
"movq (%[table], " TMP0 ", 8), %[crc1]\n"
"movzbl " BUF2L ", " TMP0W "\n"
"movq (%[table], " TMP0 ", 8), %[crc2]\n"
"movzbl " BUF3L ", " TMP0W "\n"
"movq (%[table], " TMP0 ", 8), %[crc3]\n"
#define XOR1(byte1) \
"movzbl " BUF0L ", " TMP0W "\n" \
"xorq " #byte1 "*256*8(%[table], " TMP0 ", 8), %[crc0]\n" \
"movzbl " BUF1L ", " TMP0W "\n" \
"xorq " #byte1 "*256*8(%[table], " TMP0 ", 8), %[crc1]\n" \
"movzbl " BUF2L ", " TMP0W "\n" \
"xorq " #byte1 "*256*8(%[table], " TMP0 ", 8), %[crc2]\n" \
"movzbl " BUF3L ", " TMP0W "\n" \
"xorq " #byte1 "*256*8(%[table], " TMP0 ", 8), %[crc3]\n"
#define XOR2(byte2) \
"movzbl " BUF0H ", " TMP0W "\n" \
"shrq $16, " BUF0 "\n" \
"xorq " #byte2 "*256*8(%[table], " TMP0 ", 8), %[crc0]\n" \
"movzbl " BUF1H ", " TMP0W "\n" \
"shrq $16, " BUF1 "\n" \
"xorq " #byte2 "*256*8(%[table], " TMP0 ", 8), %[crc1]\n" \
"movzbl " BUF2H ", " TMP0W "\n" \
"shrq $16, " BUF2 "\n" \
"xorq " #byte2 "*256*8(%[table], " TMP0 ", 8), %[crc2]\n" \
"movzbl " BUF3H ", " TMP0W "\n" \
"shrq $16, " BUF3 "\n" \
"xorq " #byte2 "*256*8(%[table], " TMP0 ", 8), %[crc3]\n"
XOR2(1)
XOR1(2)
XOR2(3)
XOR1(4)
XOR2(5)
XOR1(6)
// Update CRC registers and load buffers.
"movzbl " BUF0H ", " TMP0W "\n"
"xorq 7*256*8(%[table], " TMP0 ", 8), %[crc0]\n"
"movq (%[src]), " BUF0 "\n"
"movzbl " BUF1H ", " TMP0W "\n"
"xorq 7*256*8(%[table], " TMP0 ", 8), %[crc1]\n"
"movq 1*8(%[src]), " BUF1 "\n"
"movzbl " BUF2H ", " TMP0W "\n"
"xorq 7*256*8(%[table], " TMP0 ", 8), %[crc2]\n"
"movq 2*8(%[src]), " BUF2 "\n"
"movzbl " BUF3H ", " TMP0W "\n"
"xorq 7*256*8(%[table], " TMP0 ", 8), %[crc3]\n"
"movq 3*8(%[src]), " BUF3 "\n"
"cmpq %[src], %[end]\n"
"ja 1b\n"
CRC_WORD_ASM()
"xorq %[crc1], " BUF1 "\n"
"movq " BUF1 ", " BUF0 "\n"
CRC_WORD_ASM()
"xorq %[crc2], " BUF2 "\n"
"movq " BUF2 ", " BUF0 "\n"
CRC_WORD_ASM()
"xorq %[crc3], " BUF3 "\n"
"movq " BUF3 ", " BUF0 "\n"
CRC_WORD_ASM()
"add $4*8, %[src]\n"
"2:\n"
"add $2*4*8 - 8, %[end]\n"
"cmpq %[src], %[end]\n"
"jbe 4f\n"
"3:\n"
"movq (%[src]), " BUF0 "\n"
"add $8, %[src]\n"
CRC_WORD_ASM()
"cmpq %[src], %[end]\n"
"ja 3b\n"
"4:\n"
"add $7, %[end]\n"
"cmpq %[src], %[end]\n"
"jbe 6f\n"
"5:\n"
"movzbq (%[src]), " BUF0 "\n"
"movzbq %b[crc0], " TMP0 "\n"
"shrq $8, %[crc0]\n"
"xorq " BUF0 ", " TMP0 "\n"
"add $1, %[src]\n"
"xorq 7*256*8(%[table_word], " TMP0 ", 8), %[crc0]\n"
"cmpq %[src], %[end]\n"
"ja 5b\n"
"6:\n"
: // outputs
[src] "+r" (src),
[end] "+r" (end),
[crc0] "+r" (crc0),
[crc1] "=&r" (crc1),
[crc2] "=&r" (crc2),
[crc3] "=&r" (crc3)
: // inputs
[table] "r" (&this->crc_word_interleaved_[0][0]),
[table_word] "r" (&this->crc_word_[0][0])
: // clobbers
"%rax", // BUF0
"%rbx", // BUF1
"%rcx", // BUF2
"%rdx", // BUF3
"%rsi" // TMP0
);
return (crc0 ^ this->Base().Canonize());
}
} // namespace crcutil
#endif // defined(__GNUC__) && HAVE_AMD64 && CRCUTIL_USE_ASM