The Microsoft Crypto API should be quite easy to work with you would think: encrypt call, decrypt call, couple of settings. But no, there are quite a few gotchas, so heres a complete example to help you out. Just cut and paste this into your code to see it working. Oh i've removed lots of error handling for clarity, but thats pretty easy to add.

Source

#include <windows.h>
#include <wincrypt.h>
//this is keylength bit shifted, here we have keylength of 1024bits hence 1024 << 16;
#define KEYLENGTH 0x04000000
DWORD dwFlags = 0;
HCRYPTPROV m_hProv;
PBYTE pbKeyBlob;
DWORD dwKeyBlobLen;
DWORD m_dwBlockSize;
HCRYPTKEY hPubKey;

// Ok this initialises the Crypto stuff, here im using the enhanced provider
if (!CryptAcquireContext(&m_hProv, NULL, MS_ENHANCED_PROV, PROV_RSA_FULL, dwFlags))
{
   // If cant get default provider the following creates one
   DWORD err = GetLastError();
   INT iReturn = 0;
   LPSTR pszContainerName = NULL;
   DWORD cbContainerName = 0;

   if(GetLastError() != NTE_BAD_KEYSET)
   {
       printf("Error opening default key container!\r\n");
   }

   // Create default key container.
   if(!CryptAcquireContext(&m_hProv, NULL, NULL, PROV_RSA_FULL, CRYPT_NEWKEYSET))
   {
        printf("Error creating default key container!\n");
   }

   // Get size of the name of the default key container name.
   if(CryptGetProvParam(m_hProv, PP_CONTAINER, NULL, &cbContainerName, 0))
   {
      // Allocate buffer to receive default key container name.
      pszContainerName = (char *)malloc(cbContainerName);
      if(pszContainerName)
      {
         // Get name of default key container name.
        if(!CryptGetProvParam(m_hProv, PP_CONTAINER, (BYTE *)pszContainerName, &cbContainerName, 0))
        {
            // Error getting default key container name.
            pszContainerName[0] = 0;
         }
      }
   }

   // printf("Create key container '%s'\n", pszContainerName ? pszContainerName : "");

   // Free container name buffer (if created)
   if(pszContainerName)
   {
       free(pszContainerName);
   }
}

// now we can generate a key pair (that public and private)
// here we are using the RSA exchange method
BOOL bResult = CryptGenKey(m_hProv, CALG_RSA_KEYX,  KEYLENGTH | CRYPT_EXPORTABLE, &hPubKey);

// In real world usage u probably want to send the public key to your client
// so they can encrypt, we do this by exporting the public key, like so
// First get the length of the pub key
if(!CryptExportKey(hPubKey, NULL, PUBLICKEYBLOB, 0, NULL, &dwKeyBlobLen))
{
   //deal with errors (wrong types, not set to be exportable etc)
}
if((pbKeyBlob = (unsigned char *) malloc(dwKeyBlobLen)) == NULL)
{
    //alloc prob
}

// Export session key into a simple key blob.
if(!CryptExportKey(hPubKey, NULL, PUBLICKEYBLOB, 0, pbKeyBlob, &dwKeyBlobLen))
{
   // any probs
}

// Now this bit u would do on your client (after it arrived), you would init the client
as you would server, but then u would import the key
if(CryptImportKey(m_hProv,pbKeyBlob,dwKeyBlobLen,0,CRYPT_EXPORTABLE,&hPubKey) )
{
    // now we get the block length as we can only encrypt up to that size, per pass
    // this can be done as soon as u generate the key u dont need to import/export 2 get it
    if(CryptGetKeyParam(hPubKey, KP_BLOCKLEN, (LPBYTE)&m_dwKeySizeInBits, &dwLen,0))
    {
         m_dwBlockSize = m_dwKeySizeInBits/8;
    }
}

// now with RSA enc, it needs 11 bytes of padding at end, so this reduces the length further
DWORD tempLen=m_dwBlockSize-11;
BYTE *tempBuff = new BYTE[m_dwBlockSize];

// fill up our temp buffer with some data to see if we can get back
// to this when we decrypt
for(int i =0; i<m_dwBlockSize; i++)
{
     tempBuff[i] = i;
}

// ok crypt time, tempLen returns with the length of our new encrypted data
// one big note here, u cant encrypt less than 8 bytes, so u have to buffer
if(!CryptEncrypt(hPubKey, NULL, FALSE, 0, tempBuff, &tempLen, dwBlockSize))
{
   // if anything goes wrong
}

//now try and decrypt it, again we get length back of orig data
if(!CryptDecrypt(hPubKey, NULL, FALSE, 0, (BYTE *)tempBuff, &tempLen))
{
    // if anything goes wrong
}

// obviously we need to cleanup nicely
delete[] tempBuff;
CryptDestroyKey(hPubKey);
CryptReleaseContext(m_hProv,0);

Well i hope this helps get you started with Crypto API. As per normal if you spot any errors, or things change with new releases, please drop me an email at admin@niris.co.uk so i can correct it.