Untitled1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7d937360-c92b-48d7-aa88-213d46a69231",
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "cannot import name 'OpenAI' from 'openai' (C:\\Users\\The_Creater\\Documents\\llm-tutorial\\cuda\\Lib\\site-packages\\openai\\__init__.py)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[4], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mopenai\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m OpenAI\n\u001b[0;32m      2\u001b[0m OPENAI_API_KEY \u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msk-proj-fu905t6ExXMTLBiTww6ioeEkJ7ay9sxErBsM-6yUJwnOPIkajgsXWOHbEWT3BlbkFJkew6FhfxxTBFDhIn5NQwjZQuYdc8RD1SirSG7j2PGVBuL_WtTeLCs5-PAA\u001b[39m\u001b[38;5;124m\"\u001b[39m\n",
      "\u001b[1;31mImportError\u001b[0m: cannot import name 'OpenAI' from 'openai' (C:\\Users\\The_Creater\\Documents\\llm-tutorial\\cuda\\Lib\\site-packages\\openai\\__init__.py)"
     ]
    }
   ],
   "source": [
    "from openai import OpenAI\n",
    "OPENAI_API_KEY =\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bbfd4f83-55e5-42b6-ab98-3748bdc8360c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "How many files you would like to split this into? 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████| 1000/1000 [01:35<00:00, 10.47it/s]\n"
     ]
    }
   ],
   "source": [
    "import lzma\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "\n",
    "def xz_files_in_dir(directory):\n",
    "    return [filename for filename in os.listdir(directory) if filename.endswith(\".xz\") and os.path.isfile(os.path.join(directory, filename))]\n",
    "\n",
    "folder_path = \"openwebtext\"\n",
    "output_file = \"output{}.txt\"\n",
    "vocab_file = \"vocab.txt\"\n",
    "split_files = int(input(\"How many files you would like to split this into?\"))\n",
    "\n",
    "files = xz_files_in_dir(folder_path)\n",
    "total_files = len(files)\n",
    "\n",
    "# Ensure split_files is not zero to avoid division by zero\n",
    "if split_files == 0:\n",
    "    split_files = 1\n",
    "\n",
    "max_count = total_files // split_files\n",
    "vocab = set()  # A set is a collection of different items\n",
    "\n",
    "for i in range(split_files):\n",
    "    with open(output_file.format(i), \"w\", encoding=\"utf-8\") as outfile:\n",
    "        for count, filename in enumerate(tqdm(files[:max_count], total=max_count)):\n",
    "            if count >= max_count:\n",
    "                break\n",
    "            file_path = os.path.join(folder_path, filename)\n",
    "            try:\n",
    "                with lzma.open(file_path, \"rt\", encoding=\"utf-8\") as infile:\n",
    "                    text = infile.read()\n",
    "                    outfile.write(text)\n",
    "                    characters = set(text)\n",
    "                    vocab.update(characters)\n",
    "            except FileNotFoundError:\n",
    "                print(f\"File not found: {file_path}\")\n",
    "                continue\n",
    "        files = files[max_count:]\n",
    "\n",
    "with open(vocab_file, \"w\", encoding=\"utf-8\") as vfile:\n",
    "    for char in vocab:\n",
    "        vfile.write(char + '\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09c6512b-e40b-48c0-96ae-6b3c37233ae6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "cuda-gpt",
   "language": "python",
   "name": "cuda"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}