path: root/security/nss/lib/freebl/arcfour.c

/* arcfour.c - the arc four algorithm.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifdef FREEBL_NO_DEPEND
#include "stubs.h"
#endif

#include "prerr.h"
#include "secerr.h"

#include "prtypes.h"
#include "blapi.h"

/* Architecture-dependent defines */

#if defined(SOLARIS) || defined(HPUX) || defined(NSS_X86) || \
    defined(_WIN64)
/* Convert the byte-stream to a word-stream */
#define CONVERT_TO_WORDS
#endif

#if defined(AIX) || defined(OSF1) || defined(NSS_BEVAND_ARCFOUR)
/* Treat array variables as words, not bytes, on CPUs that take
 * much longer to write bytes than to write words, or when using
 * assembler code that required it.
 */
#define USE_WORD
#endif

#if defined(IS_64) || defined(NSS_BEVAND_ARCFOUR)
typedef PRUint64 WORD;
#else
typedef PRUint32 WORD;
#endif
#define WORDSIZE sizeof(WORD)

#if defined(USE_WORD)
typedef WORD Stype;
#else
typedef PRUint8 Stype;
#endif

#define ARCFOUR_STATE_SIZE 256

#define MASK1BYTE (WORD)(0xff)

#define SWAP(a, b) \
    tmp = a;       \
    a = b;         \
    b = tmp;

/*
 * State information for stream cipher.
 */
struct RC4ContextStr {
#if defined(NSS_ARCFOUR_IJ_B4_S) || defined(NSS_BEVAND_ARCFOUR)
    Stype i;
    Stype j;
    Stype S[ARCFOUR_STATE_SIZE];
#else
    Stype S[ARCFOUR_STATE_SIZE];
    Stype i;
    Stype j;
#endif
};

/*
 * array indices [0..255] to initialize cx->S array (faster than loop).
 */
static const Stype Kinit[256] = {
    0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
    0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
    0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
    0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
    0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27,
    0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f,
    0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
    0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f,
    0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
    0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f,
    0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57,
    0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f,
    0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
    0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f,
    0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
    0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f,
    0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
    0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
    0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
    0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f,
    0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
    0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
    0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7,
    0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf,
    0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7,
    0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf,
    0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7,
    0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf,
    0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7,
    0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef,
    0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7,
    0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff
};

RC4Context *
RC4_AllocateContext(void)
{
    return PORT_ZNew(RC4Context);
}

SECStatus
RC4_InitContext(RC4Context *cx, const unsigned char *key, unsigned int len,
                const unsigned char *unused1, int unused2,
                unsigned int unused3, unsigned int unused4)
{
    unsigned int i;
    PRUint8 j, tmp;
    PRUint8 K[256];
    PRUint8 *L;

    /* verify the key length. */
    PORT_Assert(len > 0 && len < ARCFOUR_STATE_SIZE);
    if (len == 0 || len >= ARCFOUR_STATE_SIZE) {
        PORT_SetError(SEC_ERROR_BAD_KEY);
        return SECFailure;
    }
    if (cx == NULL) {
        PORT_SetError(SEC_ERROR_INVALID_ARGS);
        return SECFailure;
    }
    /* Initialize the state using array indices. */
    memcpy(cx->S, Kinit, sizeof cx->S);
    /* Fill in K repeatedly with values from key. */
    L = K;
    for (i = sizeof K; i > len; i -= len) {
        memcpy(L, key, len);
        L += len;
    }
    memcpy(L, key, i);
    /* Stir the state of the generator.  At this point it is assumed
     * that the key is the size of the state buffer.  If this is not
     * the case, the key bytes are repeated to fill the buffer.
     */
    j = 0;
#define ARCFOUR_STATE_STIR(ii) \
    j = j + cx->S[ii] + K[ii]; \
    SWAP(cx->S[ii], cx->S[j]);
    for (i = 0; i < ARCFOUR_STATE_SIZE; i++) {
        ARCFOUR_STATE_STIR(i);
    }
    cx->i = 0;
    cx->j = 0;
    return SECSuccess;
}

/*
 * Initialize a new generator.
 */
RC4Context *
RC4_CreateContext(const unsigned char *key, int len)
{
    RC4Context *cx = RC4_AllocateContext();
    if (cx) {
        SECStatus rv = RC4_InitContext(cx, key, len, NULL, 0, 0, 0);
        if (rv != SECSuccess) {
            PORT_ZFree(cx, sizeof(*cx));
            cx = NULL;
        }
    }
    return cx;
}

void
RC4_DestroyContext(RC4Context *cx, PRBool freeit)
{
    if (freeit)
        PORT_ZFree(cx, sizeof(*cx));
}

#if defined(NSS_BEVAND_ARCFOUR)
extern void ARCFOUR(RC4Context *cx, WORD inputLen,
                    const unsigned char *input, unsigned char *output);
#else
/*
 * Generate the next byte in the stream.
 */
#define ARCFOUR_NEXT_BYTE() \
    tmpSi = cx->S[++tmpi];  \
    tmpj += tmpSi;          \
    tmpSj = cx->S[tmpj];    \
    cx->S[tmpi] = tmpSj;    \
    cx->S[tmpj] = tmpSi;    \
    t = tmpSi + tmpSj;

#ifdef CONVERT_TO_WORDS
/*
 * Straight ARCFOUR op.  No optimization.
 */
static SECStatus
rc4_no_opt(RC4Context *cx, unsigned char *output,
           unsigned int *outputLen, unsigned int maxOutputLen,
           const unsigned char *input, unsigned int inputLen)
{
    PRUint8 t;
    Stype tmpSi, tmpSj;
    register PRUint8 tmpi = cx->i;
    register PRUint8 tmpj = cx->j;
    unsigned int index;
    PORT_Assert(maxOutputLen >= inputLen);
    if (maxOutputLen < inputLen) {
        PORT_SetError(SEC_ERROR_OUTPUT_LEN);
        return SECFailure;
    }
    for (index = 0; index < inputLen; index++) {
        /* Generate next byte from stream. */
        ARCFOUR_NEXT_BYTE();
        /* output = next stream byte XOR next input byte */
        output[index] = cx->S[t] ^ input[index];
    }
    *outputLen = inputLen;
    cx->i = tmpi;
    cx->j = tmpj;
    return SECSuccess;
}

#else
/* !CONVERT_TO_WORDS */

/*
 * Byte-at-a-time ARCFOUR, unrolling the loop into 8 pieces.
 */
static SECStatus
rc4_unrolled(RC4Context *cx, unsigned char *output,
             unsigned int *outputLen, unsigned int maxOutputLen,
             const unsigned char *input, unsigned int inputLen)
{
    PRUint8 t;
    Stype tmpSi, tmpSj;
    register PRUint8 tmpi = cx->i;
    register PRUint8 tmpj = cx->j;
    int index;
    PORT_Assert(maxOutputLen >= inputLen);
    if (maxOutputLen < inputLen) {
        PORT_SetError(SEC_ERROR_OUTPUT_LEN);
        return SECFailure;
    }
    for (index = inputLen / 8; index-- > 0; input += 8, output += 8) {
        ARCFOUR_NEXT_BYTE();
        output[0] = cx->S[t] ^ input[0];
        ARCFOUR_NEXT_BYTE();
        output[1] = cx->S[t] ^ input[1];
        ARCFOUR_NEXT_BYTE();
        output[2] = cx->S[t] ^ input[2];
        ARCFOUR_NEXT_BYTE();
        output[3] = cx->S[t] ^ input[3];
        ARCFOUR_NEXT_BYTE();
        output[4] = cx->S[t] ^ input[4];
        ARCFOUR_NEXT_BYTE();
        output[5] = cx->S[t] ^ input[5];
        ARCFOUR_NEXT_BYTE();
        output[6] = cx->S[t] ^ input[6];
        ARCFOUR_NEXT_BYTE();
        output[7] = cx->S[t] ^ input[7];
    }
    index = inputLen % 8;
    if (index) {
        input += index;
        output += index;
        switch (index) {
            case 7:
                ARCFOUR_NEXT_BYTE();
                output[-7] = cx->S[t] ^ input[-7]; /* FALLTHRU */
            case 6:
                ARCFOUR_NEXT_BYTE();
                output[-6] = cx->S[t] ^ input[-6]; /* FALLTHRU */
            case 5:
                ARCFOUR_NEXT_BYTE();
                output[-5] = cx->S[t] ^ input[-5]; /* FALLTHRU */
            case 4:
                ARCFOUR_NEXT_BYTE();
                output[-4] = cx->S[t] ^ input[-4]; /* FALLTHRU */
            case 3:
                ARCFOUR_NEXT_BYTE();
                output[-3] = cx->S[t] ^ input[-3]; /* FALLTHRU */
            case 2:
                ARCFOUR_NEXT_BYTE();
                output[-2] = cx->S[t] ^ input[-2]; /* FALLTHRU */
            case 1:
                ARCFOUR_NEXT_BYTE();
                output[-1] = cx->S[t] ^ input[-1]; /* FALLTHRU */
            default:
                /* FALLTHRU */
                ; /* hp-ux build breaks without this */
        }
    }
    cx->i = tmpi;
    cx->j = tmpj;
    *outputLen = inputLen;
    return SECSuccess;
}
#endif

#ifdef IS_LITTLE_ENDIAN
#define ARCFOUR_NEXT4BYTES_L(n)               \
    ARCFOUR_NEXT_BYTE();                      \
    streamWord |= (WORD)cx->S[t] << (n);      \
    ARCFOUR_NEXT_BYTE();                      \
    streamWord |= (WORD)cx->S[t] << (n + 8);  \
    ARCFOUR_NEXT_BYTE();                      \
    streamWord |= (WORD)cx->S[t] << (n + 16); \
    ARCFOUR_NEXT_BYTE();                      \
    streamWord |= (WORD)cx->S[t] << (n + 24);
#else
#define ARCFOUR_NEXT4BYTES_B(n)               \
    ARCFOUR_NEXT_BYTE();                      \
    streamWord |= (WORD)cx->S[t] << (n + 24); \
    ARCFOUR_NEXT_BYTE();                      \
    streamWord |= (WORD)cx->S[t] << (n + 16); \
    ARCFOUR_NEXT_BYTE();                      \
    streamWord |= (WORD)cx->S[t] << (n + 8);  \
    ARCFOUR_NEXT_BYTE();                      \
    streamWord |= (WORD)cx->S[t] << (n);
#endif

#if (defined(IS_64) && !defined(__sparc)) || defined(NSS_USE_64)
/* 64-bit wordsize */
#ifdef IS_LITTLE_ENDIAN
#define ARCFOUR_NEXT_WORD()       \
    {                             \
        streamWord = 0;           \
        ARCFOUR_NEXT4BYTES_L(0);  \
        ARCFOUR_NEXT4BYTES_L(32); \
    }
#else
#define ARCFOUR_NEXT_WORD()       \
    {                             \
        streamWord = 0;           \
        ARCFOUR_NEXT4BYTES_B(32); \
        ARCFOUR_NEXT4BYTES_B(0);  \
    }
#endif
#else
/* 32-bit wordsize */
#ifdef IS_LITTLE_ENDIAN
#define ARCFOUR_NEXT_WORD()      \
    {                            \
        streamWord = 0;          \
        ARCFOUR_NEXT4BYTES_L(0); \
    }
#else
#define ARCFOUR_NEXT_WORD()      \
    {                            \
        streamWord = 0;          \
        ARCFOUR_NEXT4BYTES_B(0); \
    }
#endif
#endif

#ifdef IS_LITTLE_ENDIAN
#define RSH <<
#define LSH >>
#else
#define RSH >>
#define LSH <<
#endif

#ifdef IS_LITTLE_ENDIAN
#define LEFTMOST_BYTE_SHIFT 0
#define NEXT_BYTE_SHIFT(shift) shift + 8
#else
#define LEFTMOST_BYTE_SHIFT 8 * (WORDSIZE - 1)
#define NEXT_BYTE_SHIFT(shift) shift - 8
#endif

#ifdef CONVERT_TO_WORDS
static SECStatus
rc4_wordconv(RC4Context *cx, unsigned char *output,
             unsigned int *outputLen, unsigned int maxOutputLen,
             const unsigned char *input, unsigned int inputLen)
{
    PR_STATIC_ASSERT(sizeof(PRUword) == sizeof(ptrdiff_t));
    unsigned int inOffset = (PRUword)input % WORDSIZE;
    unsigned int outOffset = (PRUword)output % WORDSIZE;
    register WORD streamWord;
    register const WORD *pInWord;
    register WORD *pOutWord;
    register WORD inWord, nextInWord;
    PRUint8 t;
    register Stype tmpSi, tmpSj;
    register PRUint8 tmpi = cx->i;
    register PRUint8 tmpj = cx->j;
    unsigned int bufShift, invBufShift;
    unsigned int i;
    const unsigned char *finalIn;
    unsigned char *finalOut;

    PORT_Assert(maxOutputLen >= inputLen);
    if (maxOutputLen < inputLen) {
        PORT_SetError(SEC_ERROR_OUTPUT_LEN);
        return SECFailure;
    }
    if (inputLen < 2 * WORDSIZE) {
        /* Ignore word conversion, do byte-at-a-time */
        return rc4_no_opt(cx, output, outputLen, maxOutputLen, input, inputLen);
    }
    *outputLen = inputLen;
    pInWord = (const WORD *)(input - inOffset);
    pOutWord = (WORD *)(output - outOffset);
    if (inOffset <= outOffset) {
        bufShift = 8 * (outOffset - inOffset);
        invBufShift = 8 * WORDSIZE - bufShift;
    } else {
        invBufShift = 8 * (inOffset - outOffset);
        bufShift = 8 * WORDSIZE - invBufShift;
    }
    /*****************************************************************/
    /* Step 1:                                                       */
    /* If the first output word is partial, consume the bytes in the */
    /* first partial output word by loading one or two words of      */
    /* input and shifting them accordingly.  Otherwise, just load    */
    /* in the first word of input.  At the end of this block, at     */
    /* least one partial word of input should ALWAYS be loaded.      */
    /*****************************************************************/
    if (outOffset) {
        unsigned int byteCount = WORDSIZE - outOffset;
        for (i = 0; i < byteCount; i++) {
            ARCFOUR_NEXT_BYTE();
            output[i] = cx->S[t] ^ input[i];
        }
        /* Consumed byteCount bytes of input */
        inputLen -= byteCount;
        pInWord++;

        /* move to next word of output */
        pOutWord++;

        /* If buffers are relatively misaligned, shift the bytes in inWord
         * to be aligned to the output buffer.
         */
        if (inOffset < outOffset) {
            /* The first input word (which may be partial) has more bytes
             * than needed.  Copy the remainder to inWord.
             */
            unsigned int shift = LEFTMOST_BYTE_SHIFT;
            inWord = 0;
            for (i = 0; i < outOffset - inOffset; i++) {
                inWord |= (WORD)input[byteCount + i] << shift;
                shift = NEXT_BYTE_SHIFT(shift);
            }
        } else if (inOffset > outOffset) {
            /* Consumed some bytes in the second input word.  Copy the
             * remainder to inWord.
             */
            inWord = *pInWord++;
            inWord = inWord LSH invBufShift;
        } else {
            inWord = 0;
        }
    } else {
        /* output is word-aligned */
        if (inOffset) {
            /* Input is not word-aligned.  The first word load of input
             * will not produce a full word of input bytes, so one word
             * must be pre-loaded.  The main loop below will load in the
             * next input word and shift some of its bytes into inWord
             * in order to create a full input word.  Note that the main
             * loop must execute at least once because the input must
             * be at least two words.
             */
            unsigned int shift = LEFTMOST_BYTE_SHIFT;
            inWord = 0;
            for (i = 0; i < WORDSIZE - inOffset; i++) {
                inWord |= (WORD)input[i] << shift;
                shift = NEXT_BYTE_SHIFT(shift);
            }
            pInWord++;
        } else {
            /* Input is word-aligned.  The first word load of input
             * will produce a full word of input bytes, so nothing
             * needs to be loaded here.
             */
            inWord = 0;
        }
    }
    /*****************************************************************/
    /* Step 2: main loop                                             */
    /* At this point the output buffer is word-aligned.  Any unused  */
    /* bytes from above will be in inWord (shifted correctly).  If   */
    /* the input buffer is unaligned relative to the output buffer,  */
    /* shifting has to be done.                                      */
    /*****************************************************************/
    if (bufShift) {
        /* preloadedByteCount is the number of input bytes pre-loaded
         * in inWord.
         */
        unsigned int preloadedByteCount = bufShift / 8;
        for (; inputLen >= preloadedByteCount + WORDSIZE;
             inputLen -= WORDSIZE) {
            nextInWord = *pInWord++;
            inWord |= nextInWord RSH bufShift;
            nextInWord = nextInWord LSH invBufShift;
            ARCFOUR_NEXT_WORD();
            *pOutWord++ = inWord ^ streamWord;
            inWord = nextInWord;
        }
        if (inputLen == 0) {
            /* Nothing left to do. */
            cx->i = tmpi;
            cx->j = tmpj;
            return SECSuccess;
        }
        finalIn = (const unsigned char *)pInWord - preloadedByteCount;
    } else {
        for (; inputLen >= WORDSIZE; inputLen -= WORDSIZE) {
            inWord = *pInWord++;
            ARCFOUR_NEXT_WORD();
            *pOutWord++ = inWord ^ streamWord;
        }
        if (inputLen == 0) {
            /* Nothing left to do. */
            cx->i = tmpi;
            cx->j = tmpj;
            return SECSuccess;
        }
        finalIn = (const unsigned char *)pInWord;
    }
    /*****************************************************************/
    /* Step 3:                                                       */
    /* Do the remaining partial word of input one byte at a time.    */
    /*****************************************************************/
    finalOut = (unsigned char *)pOutWord;
    for (i = 0; i < inputLen; i++) {
        ARCFOUR_NEXT_BYTE();
        finalOut[i] = cx->S[t] ^ finalIn[i];
    }
    cx->i = tmpi;
    cx->j = tmpj;
    return SECSuccess;
}
#endif
#endif /* NSS_BEVAND_ARCFOUR */

SECStatus
RC4_Encrypt(RC4Context *cx, unsigned char *output,
            unsigned int *outputLen, unsigned int maxOutputLen,
            const unsigned char *input, unsigned int inputLen)
{
    PORT_Assert(maxOutputLen >= inputLen);
    if (maxOutputLen < inputLen) {
        PORT_SetError(SEC_ERROR_OUTPUT_LEN);
        return SECFailure;
    }
#if defined(NSS_BEVAND_ARCFOUR)
    ARCFOUR(cx, inputLen, input, output);
    *outputLen = inputLen;
    return SECSuccess;
#elif defined(CONVERT_TO_WORDS)
    /* Convert the byte-stream to a word-stream */
    return rc4_wordconv(cx, output, outputLen, maxOutputLen, input, inputLen);
#else
    /* Operate on bytes, but unroll the main loop */
    return rc4_unrolled(cx, output, outputLen, maxOutputLen, input, inputLen);
#endif
}

SECStatus
RC4_Decrypt(RC4Context *cx, unsigned char *output,
            unsigned int *outputLen, unsigned int maxOutputLen,
            const unsigned char *input, unsigned int inputLen)
{
    PORT_Assert(maxOutputLen >= inputLen);
    if (maxOutputLen < inputLen) {
        PORT_SetError(SEC_ERROR_OUTPUT_LEN);
        return SECFailure;
    }
/* decrypt and encrypt are same operation. */
#if defined(NSS_BEVAND_ARCFOUR)
    ARCFOUR(cx, inputLen, input, output);
    *outputLen = inputLen;
    return SECSuccess;
#elif defined(CONVERT_TO_WORDS)
    /* Convert the byte-stream to a word-stream */
    return rc4_wordconv(cx, output, outputLen, maxOutputLen, input, inputLen);
#else
    /* Operate on bytes, but unroll the main loop */
    return rc4_unrolled(cx, output, outputLen, maxOutputLen, input, inputLen);
#endif
}

#undef CONVERT_TO_WORDS
#undef USE_WORD