Alright, a friend of mine wrote a simple encryption scheme...much like the one i posted not too long ago. I've been attempting to crack it, and don't think I completely understand how to do so. I'll post his src, and my crack. As a side note, I've been challenged to crack it. I have one month, and I cannot brute force.

Encryption Scheme:

Code:
#include <iostream.h>  //for cin and cout
#include <string.h>  //for strcpy()
#include <fstream.h>  //for ifstream and ofstream
#include <stdlib.h>  //for system() and srand() and rand()
#include <sys/stat.h>  //for stat() and stat structure

int main(void)
{
    char file[1024];  //to hold file path
    cout << "File to Encrypt: " << flush;
    cin.getline(file, 1023); //input file path from user
    struct stat result; //this will hold the the file's size
    stat(file, &result); //get the file's size from it's path, and place it in the result structure
    unsigned long flen = result.st_size;  //set our local variable to the size of the file
    cout << flen << " Bytes.  Enter 1 if correct: " << flush;
    unsigned long key, choice;
    cin >> choice;
    if (choice != 1)  //verify that data is correct before continueing
    {
        return 0;
    }
    cout << "Key: " << flush;
    cin >> key;  //input the encryption key
    cout << "Enter 1 if you are encrypting, 0 if decrypting: " << flush;
    cin >> choice;
    bool e_or_d;
    switch (choice)  //set encrypt or decrypt
    {
        case 0: e_or_d = false;
                break;
        case 1: e_or_d = true;
                break;
        default: return 0;
    }    
    system("cls");  //clear the screen
    cout << "Verify" << endl
         << "------" << endl
         << "File: " << file << endl
         << "Size: " << flen << endl
         << "Key : " << key << endl;
    if (e_or_d)
    {
        cout << "Encrypting" << endl;
    }
    else
    {
        cout << "Decrypting" << endl;
    }
    cout << endl
         << "Enter 1 if this is correct: " << flush;
    cin >> choice;
    if (choice == 0)  //verify that all data is correct again
    {
        return 0;
    }
    system("cls");
    cout << "Encrypting, standbye..." << flush;
    ifstream in(file, ios::binary);  //open our file
    strcpy(&file[strlen(file)], ".eng"); //this will turn something like C:\hello.txt into C:\hello.txt.eng
    ofstream out(file, ios::binary | ios::trunc);  //open our output file
    char buf;  //our buffer
    unsigned long x;  //counter variable
    srand(key); //randomize by the key
    for (x = 0; x < flen; x++) //loop until we've reached all bytes
    {
        in.get(buf);  //get 1 byte from file
        if (e_or_d)
            buf += (rand() % 713) - 331;  //encrypt it using this function.  To decrypt it's -=
        else
            buf -= (rand() % 713) - 331;  //decrypt it
        out.put(buf);  //output the new byte
    }
    out.flush();  //make sure everything has been written to the disk
    in.close();  //close file
    out.close();  //close file
    cout << "done" << endl << endl
         << "Saved as " << file << endl << endl
         << "Goodbye\n" << flush;  //closing message
    system("pause");  //pause
    return 0; //end program
}
He encrypted a file called hellokittyenc.txt. I am attempting to crack it, and output to hellokitty.txt. I assumed that all of the letters in his file (it is plaintext) would all have the same number added to it, and would be one of the 256 characters in the ASCII table. I don't understand why my crack doesn't work. The only things that come to mind are problems with character translations opening that TXT in binary mode...but he has successfully encrypted/decrypted it with the key. Any suggestions?

NOTE: While running the program, I pause every 20 or so, and view the output to see if it's turned into plain english, if not I resume for another 20 or so. I have to stop every once in a while, because the cmd screen will not scroll all the way back up to view the first like 200 attempts.

Code:
#include <iostream.h>
#include <fstream.h>



char tmp;
int x = 0;

main()
{
    for(x = 0; x <= 255; x++)
    {
        ofstream out("hellokitty.txt", ios::binary | ios::trunc);
        ifstream in("hellokittyenc.txt", ios::binary);
        
        while(!in.eof())
        {
                in.get(tmp);
                tmp -= x;
                out.put(tmp);
        }
        
        out.flush();
        in.close();
        out.close();
        cout << "Code: " << x << endl;
        system("type hellokitty.txt");
        cout << endl;
        cout << "Attempted Decryption Complete\n";
        cout << endl;
    }
    
    system("pause");
        
}