17 #if defined(CRYPTOPP_DEBUG)
19 # include "threefish.h"
23 #if defined(__SSE2__) || defined(_M_X64)
24 # include <emmintrin.h>
26 # define M128_CAST(x) ((__m128i *)(void *)(x))
27 # define CONST_M128_CAST(x) ((const __m128i *)(const void *)(x))
31 #if defined(__aarch32__) || defined(__aarch64__) || defined(_M_ARM64)
32 # if (CRYPTOPP_ARM_NEON_HEADER)
33 # include <arm_neon.h>
37 #if defined(__ALTIVEC__)
41 ANONYMOUS_NAMESPACE_BEGIN
45 #if defined(CRYPTOPP_DEBUG) && !defined(CRYPTOPP_DOXYGEN_PROCESSING)
48 using CryptoPP::XTS_Mode;
49 using CryptoPP::Threefish512;
51 void Modes_TestInstantiations()
53 XTS_Mode<AES>::Encryption m0;
54 XTS_Mode<AES>::Decryption m1;
55 XTS_Mode<AES>::Encryption m2;
56 XTS_Mode<AES>::Decryption m3;
58 #if CRYPTOPP_XTS_WIDE_BLOCK_CIPHERS
59 XTS_Mode<Threefish512>::Encryption m4;
60 XTS_Mode<Threefish512>::Decryption m5;
63 #endif // CRYPTOPP_DEBUG
65 inline void XorBuffer(
byte *output,
const byte *input,
const byte *mask,
size_t count)
69 #if defined(CRYPTOPP_DISABLE_ASM)
70 xorbuf(output, input, mask, count);
72 #elif defined(__SSE2__) || defined(_M_X64)
73 for (
size_t i=0; i<count; i+=16)
79 #elif defined(__aarch32__) || defined(__aarch64__) || defined(_M_ARM64)
80 for (
size_t i=0; i<count; i+=16)
81 vst1q_u8(output+i, veorq_u8(vld1q_u8(input+i), vld1q_u8(mask+i)));
83 #elif defined(__ALTIVEC__)
84 for (
size_t i=0; i<count; i+=16)
88 xorbuf(output, input, mask, count);
92 inline void XorBuffer(
byte *buf,
const byte *mask,
size_t count)
94 XorBuffer(buf, buf, mask, count);
98 inline void GF_Double(
byte *out,
const byte* in,
unsigned int len)
100 #if defined(_M_X64) || defined(_M_ARM64) || defined(_LP64) || defined(__LP64__)
102 for (
size_t i=0, idx=0; i<len/8; ++i, idx+=8)
105 word64 y = (x >> 63); x = (x << 1) + carry;
111 for (
size_t i=0, idx=0; i<len/4; ++i, idx+=4)
114 word32 y = (x >> 31); x = (x << 1) + carry;
120 #if CRYPTOPP_XTS_WIDE_BLOCK_CIPHERS
133 const size_t LEIDX = 16-1;
141 const size_t LEIDX = 32-1;
150 const size_t LEIDX = 64-1;
159 const size_t LEIDX = 128-1;
161 k[LEIDX-126] ^= 0x00;
162 k[LEIDX-127] ^= 0x43;
178 #endif // CRYPTOPP_XTS_WIDE_BLOCK_CIPHERS
181 inline void GF_Double(
byte *inout,
unsigned int len)
183 GF_Double(inout, inout, len);
186 ANONYMOUS_NAMESPACE_END
192 #if CRYPTOPP_XTS_WIDE_BLOCK_CIPHERS
194 if (length < 16 || length > 128 || !
IsPowerOf2(length))
195 throw InvalidArgument(AlgorithmName() +
": block size of underlying block cipher is not valid");
199 throw InvalidArgument(AlgorithmName() +
": block size of underlying block cipher is not 16");
206 if (!GetBlockCipher().IsValidKeyLength((length+1)/2))
212 ThrowIfInvalidKeyLength(length);
215 const size_t klen = length/2;
216 AccessBlockCipher().SetKey(key+0, klen, params);
217 AccessTweakCipher().SetKey(key+klen, klen, params);
222 const byte *iv = GetIVAndThrowIfInvalid(params, ivLength);
223 Resynchronize(iv, (
int)ivLength);
229 std::memcpy(m_xregister, m_register, ivLength);
230 GetTweakCipher().ProcessBlock(m_xregister);
236 PutWord<word64>(
false, order, iv, sector);
237 std::memset(iv+8, 0x00, iv.
size()-8);
240 std::memcpy(m_xregister, iv, iv.
size());
241 GetTweakCipher().ProcessBlock(m_xregister);
244 void XTS_ModeBase::ResizeBuffers()
246 BlockOrientedCipherModeBase::ResizeBuffers();
247 m_xworkspace.New(GetBlockCipher().
BlockSize()*ParallelBlocks);
248 m_xregister.New(GetBlockCipher().
BlockSize()*ParallelBlocks);
261 enum { lastParallelBlock = ParallelBlocks-1 };
262 const unsigned int blockSize = GetBlockCipher().BlockSize();
263 const size_t parallelSize = blockSize*ParallelBlocks;
266 while (length >= parallelSize)
269 GF_Double(m_xregister+1*blockSize, m_xregister+0*blockSize, blockSize);
270 GF_Double(m_xregister+2*blockSize, m_xregister+1*blockSize, blockSize);
271 GF_Double(m_xregister+3*blockSize, m_xregister+2*blockSize, blockSize);
273 if (ParallelBlocks > 4)
275 GF_Double(m_xregister+4*blockSize, m_xregister+3*blockSize, blockSize);
276 GF_Double(m_xregister+5*blockSize, m_xregister+4*blockSize, blockSize);
277 GF_Double(m_xregister+6*blockSize, m_xregister+5*blockSize, blockSize);
278 GF_Double(m_xregister+7*blockSize, m_xregister+6*blockSize, blockSize);
280 if (ParallelBlocks > 8)
282 GF_Double(m_xregister+8*blockSize, m_xregister+7*blockSize, blockSize);
283 GF_Double(m_xregister+9*blockSize, m_xregister+8*blockSize, blockSize);
284 GF_Double(m_xregister+10*blockSize, m_xregister+9*blockSize, blockSize);
285 GF_Double(m_xregister+11*blockSize, m_xregister+10*blockSize, blockSize);
289 XorBuffer(m_xworkspace, inString, m_xregister, parallelSize);
292 GetBlockCipher().AdvancedProcessBlocks(m_xworkspace, m_xregister,
296 GF_Double(m_xregister+0, m_xregister+lastParallelBlock*blockSize, blockSize);
298 inString += parallelSize;
299 outString += parallelSize;
300 length -= parallelSize;
304 while (ParallelBlocks == 12 && length >= blockSize*4)
307 GF_Double(m_xregister+1*blockSize, m_xregister+0*blockSize, blockSize);
308 GF_Double(m_xregister+2*blockSize, m_xregister+1*blockSize, blockSize);
309 GF_Double(m_xregister+3*blockSize, m_xregister+2*blockSize, blockSize);
312 XorBuffer(m_xworkspace, inString, m_xregister, blockSize*4);
315 GetBlockCipher().AdvancedProcessBlocks(m_xworkspace, m_xregister,
319 GF_Double(m_xregister+0, m_xregister+3*blockSize, blockSize);
321 inString += blockSize*4;
322 outString += blockSize*4;
323 length -= blockSize*4;
327 while (ParallelBlocks == 8 && length >= blockSize*2)
330 GF_Double(m_xregister+1*blockSize, m_xregister+0*blockSize, blockSize);
333 XorBuffer(m_xworkspace, inString, m_xregister, blockSize*2);
336 GetBlockCipher().AdvancedProcessBlocks(m_xworkspace, m_xregister,
340 GF_Double(m_xregister+0, m_xregister+1*blockSize, blockSize);
342 inString += blockSize*2;
343 outString += blockSize*2;
344 length -= blockSize*2;
351 XorBuffer(m_xworkspace, inString, m_xregister, blockSize);
354 GetBlockCipher().ProcessBlock(m_xworkspace);
357 XorBuffer(outString, m_xworkspace, m_xregister, blockSize);
360 GF_Double(m_xregister, blockSize);
362 inString += blockSize;
363 outString += blockSize;
374 throw InvalidArgument(
"XTS: message is too short for ciphertext stealing");
376 if (IsForwardTransformation())
377 return ProcessLastPlainBlock(outString, outLength, inString, inLength);
379 return ProcessLastCipherBlock(outString, outLength, inString, inLength);
382 size_t XTS_ModeBase::ProcessLastPlainBlock(
byte *outString,
size_t outLength,
const byte *inString,
size_t inLength)
387 const unsigned int blockSize = GetBlockCipher().BlockSize();
388 const size_t blocks = inLength / blockSize;
389 const size_t tail = inLength % blockSize;
390 outLength = inLength;
395 ProcessData(outString, inString, inLength);
401 const size_t head = (blocks-1)*blockSize;
402 ProcessData(outString, inString, inLength-head);
405 inString += head; inLength -= head;
411 XorBuffer(m_xworkspace, inString, m_xregister, blockSize);
414 GetBlockCipher().ProcessBlock(m_xworkspace);
417 XorBuffer(outString, m_xworkspace, m_xregister, blockSize);
420 GF_Double(m_xregister, blockSize);
424 inString += blockSize;
425 outString += blockSize;
426 const size_t len = inLength-blockSize;
429 std::memcpy(m_xworkspace, inString, len);
431 std::memcpy(outString, outString-blockSize, len);
433 std::memcpy(m_xworkspace+len, outString-blockSize+len, blockSize-len);
436 XorBuffer(m_xworkspace, m_xregister, blockSize);
439 GetBlockCipher().ProcessBlock(m_xworkspace);
442 XorBuffer(outString-blockSize, m_xworkspace, m_xregister, blockSize);
447 size_t XTS_ModeBase::ProcessLastCipherBlock(
byte *outString,
size_t outLength,
const byte *inString,
size_t inLength)
452 const unsigned int blockSize = GetBlockCipher().BlockSize();
453 const size_t blocks = inLength / blockSize;
454 const size_t tail = inLength % blockSize;
455 outLength = inLength;
460 ProcessData(outString, inString, inLength);
466 const size_t head = (blocks-1)*blockSize;
467 ProcessData(outString, inString, inLength-head);
470 inString += head; inLength -= head;
473 #define poly1 (m_xregister+0*blockSize)
474 #define poly2 (m_xregister+1*blockSize)
475 GF_Double(poly2, poly1, blockSize);
479 inString += blockSize;
480 outString += blockSize;
481 const size_t len = inLength-blockSize;
484 XorBuffer(m_xworkspace, inString-blockSize, poly2, blockSize);
487 GetBlockCipher().ProcessBlock(m_xworkspace);
490 XorBuffer(m_xworkspace, poly2, blockSize);
493 std::memcpy(outString-blockSize, inString, len);
495 std::memcpy(outString, m_xworkspace, len);
497 std::memcpy(outString-blockSize+len, m_xworkspace+len, blockSize-len);
501 inString -= blockSize;
502 outString -= blockSize;
505 XorBuffer(m_xworkspace, outString, poly1, blockSize);
508 GetBlockCipher().ProcessBlock(m_xworkspace);
511 XorBuffer(outString, m_xworkspace, poly1, blockSize);