diff --git a/GPT2_GPTQ_4bit.ipynb b/GPT2_GPTQ_4bit.ipynb deleted file mode 100644 index 2ad28ae..0000000 --- a/GPT2_GPTQ_4bit.ipynb +++ /dev/null @@ -1,1072 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [], - "gpuType": "T4", - "authorship_tag": "ABX9TyNC/p+CjyumuththfkLa9LG", - "include_colab_link": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - }, - "accelerator": "GPU", - "widgets": { - "application/vnd.jupyter.widget-state+json": { - "9fdc7832238743f384543674f57a135d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_e856dd2f68714377b76493d5f428043d", - "IPY_MODEL_b45fac85d2034fcda9427c787124788d", - "IPY_MODEL_8ae387bcc1a3478eb5da50db9449e7a0" - ], - "layout": "IPY_MODEL_653f5250de60427cb870fe823937e6af" - } - }, - "e856dd2f68714377b76493d5f428043d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_06846d39e16d4c66bc4cc177666c959b", - "placeholder": "​", - "style": "IPY_MODEL_0fcfa8e7621647abae79076a7aec2972", - "value": "Upload 1 LFS files: 100%" - } - }, - "b45fac85d2034fcda9427c787124788d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_811878ae336b4d0c9ec8237fc37bb999", - "max": 1, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_0146ec198b8640b2aa30f8466e40597d", - "value": 1 - } - }, - "8ae387bcc1a3478eb5da50db9449e7a0": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_769dd69f4f6f4af08d1b2d15522940ec", - "placeholder": "​", - "style": "IPY_MODEL_f32db67266234c34aa77317776cbdc48", - "value": " 1/1 [00:13<00:00, 13.70s/it]" - } - }, - "653f5250de60427cb870fe823937e6af": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "06846d39e16d4c66bc4cc177666c959b": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "0fcfa8e7621647abae79076a7aec2972": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "811878ae336b4d0c9ec8237fc37bb999": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "0146ec198b8640b2aa30f8466e40597d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "769dd69f4f6f4af08d1b2d15522940ec": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "f32db67266234c34aa77317776cbdc48": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "66a41698432f46de9eb325447917a389": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_ca0783bbd2e1428d92ee78db13d0d64d", - "IPY_MODEL_ebbb561fa4cb42e1be414b6462949fd0", - "IPY_MODEL_d2ee702c234b4b0f903e92e09cb1c6dd" - ], - "layout": "IPY_MODEL_8d25924e581e40a9a39a87d4d3d14221" - } - }, - "ca0783bbd2e1428d92ee78db13d0d64d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_2e97ccd3f59447db9892c3819252c57b", - "placeholder": "​", - "style": "IPY_MODEL_1dbd60f5ee294c65b0bb9225d81bd8af", - "value": "gptq_model-4bit-128g.bin: 100%" - } - }, - "ebbb561fa4cb42e1be414b6462949fd0": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_b0a9de27e4074778ab5a55a1f9d250cc", - "max": 123495975, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_bc79604dbd3242bab577154cca421b83", - "value": 123495975 - } - }, - "d2ee702c234b4b0f903e92e09cb1c6dd": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_8f3eab81268943a4935e04e502f95604", - "placeholder": "​", - "style": "IPY_MODEL_6e4ed763e8c843dbbdbfd5ad0570d884", - "value": " 123M/123M [00:13<00:00, 12.0MB/s]" - } - }, - "8d25924e581e40a9a39a87d4d3d14221": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "2e97ccd3f59447db9892c3819252c57b": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "1dbd60f5ee294c65b0bb9225d81bd8af": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "b0a9de27e4074778ab5a55a1f9d250cc": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "bc79604dbd3242bab577154cca421b83": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "8f3eab81268943a4935e04e502f95604": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "6e4ed763e8c843dbbdbfd5ad0570d884": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - } - } - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "view-in-github", - "colab_type": "text" - }, - "source": [ - "\"Open" - ] - }, - { - "cell_type": "markdown", - "source": [ - "# Create a 4-bit GPT-2 model using AutoGPTQ\n", - "> 🗣️ [Large Language Model Course](https://github.com/mlabonne/llm-course)\n", - "\n", - "❤️ Created by [@maximelabonne](https://twitter.com/maximelabonne).\n", - "\n", - "## Quantize model" - ], - "metadata": { - "id": "yezrHxYvg_wR" - } - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "BhufqqQAaz6e" - }, - "outputs": [], - "source": [ - "!BUILD_CUDA_EXT=0 pip install -q auto-gptq transformers huggingface_hub" - ] - }, - { - "cell_type": "code", - "source": [ - "import torch\n", - "from auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfig\n", - "from transformers import AutoTokenizer\n", - "\n", - "examples = [\n", - " \"In the wake of the Federal Reserve's recent decision, market analysts predict a shift in the stock market dynamics, urging investors to reassess their portfolios.\",\n", - " \"As quantum computing continues its rapid development, it promises to revolutionize fields such as cryptography and machine learning, posing a significant leap from classical computing.\",\n", - " \"The recent elections have brought a seismic shift in the political landscape, with the newly elected government pledging to focus on healthcare and education reform.\",\n", - " \"The Renaissance, a significant period in European history, was marked by a cultural rebirth and dramatic advances in art, science, and philosophical thought.\",\n", - " \"With the rise of machine learning and AI, Python has emerged as a dominant language in programming due to its simplicity and powerful libraries such as TensorFlow and PyTorch.\",\n", - " \"Jane Austen's 'Pride and Prejudice' continues to captivate readers with its intricate exploration of societal norms and the complexities of human relationships during the Regency era.\",\n", - " \"Following an intense season, the Golden State Warriors have emerged as the NBA champions, underscoring their remarkable team play and strategic finesse.\",\n", - " \"The latest Marvel film, 'Avengers: Infinity Gauntlet', has shattered box office records worldwide, reinforcing the global appeal of superhero narratives.\",\n", - " \"The increasing instances of wildfires and erratic weather patterns underscore the urgent need to address climate change and implement sustainable environmental practices.\",\n", - " \"In recent news, a breakthrough in the peace negotiations between the two countries has sparked hope for an end to the decade-long conflict.\",\n", - "]\n", - "\n", - "# Define base model and output directory\n", - "model_id = \"gpt2\"\n", - "out_dir = model_id + \"-GPTQ\"\n", - "\n", - "# Load quantize config, model and tokenizer\n", - "quantize_config = BaseQuantizeConfig(bits=4, group_size=128)\n", - "model = AutoGPTQForCausalLM.from_pretrained(model_id, quantize_config)\n", - "tokenizer = AutoTokenizer.from_pretrained(model_id)\n", - "\n", - "# Determine device\n", - "device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\n", - "\n", - "# Tokenize examples\n", - "examples_ids = [tokenizer(text, truncation=True) for text in examples]\n", - "\n", - "# Quantize\n", - "model.quantize(\n", - " examples_ids,\n", - " use_triton=True,\n", - " autotune_warmup_after_quantized=True,\n", - " batch_size=1,\n", - ")\n", - "\n", - "# Save model and tokenizer\n", - "model.save_quantized(model_id + \"-GPTQ\", use_safetensors=False)\n", - "model.save_quantized(model_id + \"-GPTQ\", use_safetensors=True)\n", - "tokenizer.save_pretrained(out_dir)" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "ETsG2iYrXaUg", - "outputId": "322feb57-c4bf-48aa-d29b-b71738e3edf1" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "WARNING:auto_gptq.modeling._utils:using autotune_warmup will move model to GPU, make sure you have enough VRAM to load the whole model.\n", - "100%|██████████| 11/11 [03:16<00:00, 17.87s/it]\n" - ] - }, - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "('gpt2-GPTQ/tokenizer_config.json',\n", - " 'gpt2-GPTQ/special_tokens_map.json',\n", - " 'gpt2-GPTQ/vocab.json',\n", - " 'gpt2-GPTQ/merges.txt',\n", - " 'gpt2-GPTQ/added_tokens.json',\n", - " 'gpt2-GPTQ/tokenizer.json')" - ] - }, - "metadata": {}, - "execution_count": 2 - } - ] - }, - { - "cell_type": "code", - "source": [ - "# Reload model and tokenizer\n", - "model = AutoGPTQForCausalLM.from_quantized(\n", - " out_dir,\n", - " use_triton=True,\n", - " device=device,\n", - " use_safetensors=True,\n", - ")\n", - "tokenizer = AutoTokenizer.from_pretrained(out_dir)" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "nktu1FsdZ9sd", - "outputId": "8f0aaf4e-5fc5-42d1-eb33-220658edb8d0" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "WARNING:accelerate.utils.modeling:The safetensors archive passed at gpt2-GPTQ/gptq_model-4bit-128g.safetensors does not contain metadata. Make sure to save your model with the `save_pretrained` method. Defaulting to 'pt' metadata.\n", - "WARNING:auto_gptq.modeling._base:GPT2GPTQForCausalLM hasn't fused attention module yet, will skip inject fused attention.\n", - "WARNING:auto_gptq.modeling._base:GPT2GPTQForCausalLM hasn't fused mlp module yet, will skip inject fused mlp.\n" - ] - } - ] - }, - { - "cell_type": "code", - "source": [ - "def generate_text(input_text):\n", - " input_ids = tokenizer.encode(input_text, return_tensors='pt').to(device)\n", - " attention_mask = torch.ones(input_ids.shape, dtype=torch.long).to(device)\n", - "\n", - " output = model.to(device).generate(\n", - " inputs=input_ids,\n", - " attention_mask=attention_mask,\n", - " do_sample=True,\n", - " max_length=50,\n", - " top_k=50,\n", - " pad_token_id=tokenizer.eos_token_id\n", - " )\n", - " output = tokenizer.decode(output[0], skip_special_tokens=True)\n", - "\n", - " return output\n", - "\n", - "# Generate text\n", - "input_text = \"I have a dream\"\n", - "generate_text(input_text)" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 36 - }, - "id": "KSIHpQ4XZ_7R", - "outputId": "e6f5c8a5-e3bf-4e52-d239-6b6f190e5475" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "'I have a dream,,,,,,,,, at,--,,,,,,,,,,,,,,---,,,, ( (,//,,,,---'" - ], - "application/vnd.google.colaboratory.intrinsic+json": { - "type": "string" - } - }, - "metadata": {}, - "execution_count": 24 - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "## Save and load model using Hugging Face Hub" - ], - "metadata": { - "id": "gV8hqGdYhLQH" - } - }, - { - "cell_type": "code", - "source": [ - "from huggingface_hub import notebook_login\n", - "from huggingface_hub import HfApi\n", - "import locale\n", - "locale.getpreferredencoding = lambda: \"UTF-8\"\n", - "\n", - "REPO_ID = \"insert your repo/model ID\" # example: \"mlabonne/gpt2-GPTQ-4bit\"\n", - "\n", - "notebook_login()\n", - "api = HfApi()\n", - "!git config --global credential.helper store\n", - "\n", - "api.upload_folder(\n", - " folder_path=out_dir,\n", - " repo_id=REPO_ID,\n", - " repo_type=\"model\",\n", - ")" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 100, - "referenced_widgets": [ - "9fdc7832238743f384543674f57a135d", - "e856dd2f68714377b76493d5f428043d", - "b45fac85d2034fcda9427c787124788d", - "8ae387bcc1a3478eb5da50db9449e7a0", - "653f5250de60427cb870fe823937e6af", - "06846d39e16d4c66bc4cc177666c959b", - "0fcfa8e7621647abae79076a7aec2972", - "811878ae336b4d0c9ec8237fc37bb999", - "0146ec198b8640b2aa30f8466e40597d", - "769dd69f4f6f4af08d1b2d15522940ec", - "f32db67266234c34aa77317776cbdc48", - "66a41698432f46de9eb325447917a389", - "ca0783bbd2e1428d92ee78db13d0d64d", - "ebbb561fa4cb42e1be414b6462949fd0", - "d2ee702c234b4b0f903e92e09cb1c6dd", - "8d25924e581e40a9a39a87d4d3d14221", - "2e97ccd3f59447db9892c3819252c57b", - "1dbd60f5ee294c65b0bb9225d81bd8af", - "b0a9de27e4074778ab5a55a1f9d250cc", - "bc79604dbd3242bab577154cca421b83", - "8f3eab81268943a4935e04e502f95604", - "6e4ed763e8c843dbbdbfd5ad0570d884" - ] - }, - "id": "OKTxY6jQaDMv", - "outputId": "f786a135-ca33-486f-88b8-9795fdb8e713" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "display_data", - "data": { - "text/plain": [ - "Upload 1 LFS files: 0%| | 0/1 [00:00