cookbook/Claude_(Anthropic)_with_Streaming_liteLLM_Examples.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZwuaylskLxFu",
        "outputId": "d684d6a3-32fe-4beb-c378-c39134bcf8cc"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting litellm==0.1.363\n",
            "  Downloading litellm-0.1.363-py3-none-any.whl (34 kB)\n",
            "Requirement already satisfied: openai<0.28.0,>=0.27.8 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (0.27.8)\n",
            "Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (1.0.0)\n",
            "Requirement already satisfied: tiktoken<0.5.0,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (0.4.0)\n",
            "Requirement already satisfied: requests>=2.20 in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (2.31.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (4.65.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.8.5)\n",
            "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken<0.5.0,>=0.4.0->litellm==0.1.363) (2022.10.31)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.2.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.26.16)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (2023.7.22)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (23.1.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (4.0.2)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.4.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.3.1)\n",
            "Installing collected packages: litellm\n",
            "  Attempting uninstall: litellm\n",
            "    Found existing installation: litellm 0.1.362\n",
            "    Uninstalling litellm-0.1.362:\n",
            "      Successfully uninstalled litellm-0.1.362\n",
            "Successfully installed litellm-0.1.363\n"
          ]
        }
      ],
      "source": [
        "!pip install litellm==\"0.1.363\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "W216G__XL19Q"
      },
      "outputs": [],
      "source": [
        "# @title Import litellm & Set env variables\n",
        "import litellm\n",
        "import os\n",
        "\n",
        "os.environ[\"ANTHROPIC_API_KEY\"] = \" \" #@param"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ff1lKwUMMLJj",
        "outputId": "bfddf6f8-36d4-45e5-92dc-349083fa41b8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Result from claude-instant-1 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \" The Los Angeles Dodgers won the 2020 World Series, defeating the Tampa Bay Rays 4-2. It was the Dodgers' first World Series title since 1988.\"}}], 'created': 1691536677.2676156, 'model': 'claude-instant-1', 'usage': {'prompt_tokens': 30, 'completion_tokens': 32, 'total_tokens': 62}}\n",
            "\n",
            "\n",
            " Result from claude-2 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': ' The Los Angeles Dodgers won'}}], 'created': 1691536677.944753, 'model': 'claude-2', 'usage': {'prompt_tokens': 30, 'completion_tokens': 5, 'total_tokens': 35}}\n"
          ]
        }
      ],
      "source": [
        "# @title Request Claude Instant-1 and Claude-2\n",
        "messages = [\n",
        "  {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
        "  {\"role\": \"user\", \"content\": \"Who won the world series in 2020?\"}\n",
        "  ]\n",
        "\n",
        "result = litellm.completion('claude-instant-1', messages)\n",
        "print(\"\\n\\n Result from claude-instant-1\", result)\n",
        "result = litellm.completion('claude-2', messages, max_tokens=5, temperature=0.2)\n",
        "print(\"\\n\\n Result from claude-2\", result)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "06hWKnNQMrV-",
        "outputId": "7fdec0eb-d4a9-4882-f9c4-987ff9a31114"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " Here\n",
            "'s\n",
            " a\n",
            " quick\n",
            " overview\n",
            " of\n",
            " how\n",
            " a\n",
            " court\n",
            " case\n",
            " can\n",
            " reach\n",
            " the\n",
            " U\n",
            ".\n",
            "S\n",
            ".\n",
            " Supreme\n",
            " Court\n",
            ":\n",
            "\n",
            "\n",
            "-\n",
            " The\n",
            " case\n",
            " must\n",
            " first\n",
            " be\n",
            " heard\n",
            " in\n",
            " a\n",
            " lower\n",
            " trial\n",
            " court\n",
            " (\n",
            "either\n",
            " a\n",
            " state\n",
            " court\n",
            " or\n",
            " federal\n",
            " district\n",
            " court\n",
            ").\n",
            " The\n",
            " trial\n",
            " court\n",
            " makes\n",
            " initial\n",
            " r\n",
            "ulings\n",
            " and\n",
            " produces\n",
            " a\n",
            " record\n",
            " of\n",
            " the\n",
            " case\n",
            ".\n",
            "\n",
            "\n",
            "-\n",
            " The\n",
            " losing\n",
            " party\n",
            " can\n",
            " appeal\n",
            " the\n",
            " decision\n",
            " to\n",
            " an\n",
            " appeals\n",
            " court\n",
            " (\n",
            "a\n",
            " state\n",
            " appeals\n",
            " court\n",
            " for\n",
            " state\n",
            " cases\n",
            ",\n",
            " or\n",
            " a\n",
            " federal\n",
            " circuit\n",
            " court\n",
            " for\n",
            " federal\n",
            " cases\n",
            ").\n",
            " The\n",
            " appeals\n",
            " court\n",
            " reviews\n",
            " the\n",
            " trial\n",
            " court\n",
            "'s\n",
            " r\n",
            "ulings\n",
            " and\n",
            " can\n",
            " affirm\n",
            ",\n",
            " reverse\n",
            ",\n",
            " or\n",
            " modify\n",
            " the\n",
            " decision\n",
            ".\n",
            "\n",
            "\n",
            "-\n",
            " If\n",
            " a\n",
            " party\n",
            " is\n",
            " still\n",
            " unsat\n",
            "isf\n",
            "ied\n",
            " after\n",
            " the\n",
            " appeals\n",
            " court\n",
            " rules\n",
            ",\n",
            " they\n",
            " can\n",
            " petition\n",
            " the\n",
            " Supreme\n",
            " Court\n",
            " to\n",
            " hear\n",
            " the\n",
            " case\n",
            " through\n",
            " a\n",
            " writ\n",
            " of\n",
            " cert\n",
            "ior\n",
            "ari\n",
            ".\n",
            " \n",
            "\n",
            "\n",
            "-\n",
            " The\n",
            " Supreme\n",
            " Court\n",
            " gets\n",
            " thousands\n",
            " of\n",
            " cert\n",
            " petitions\n",
            " every\n",
            " year\n",
            " but\n",
            " usually\n",
            " only\n",
            " agrees\n",
            " to\n",
            " hear\n",
            " about\n",
            " 100\n",
            "-\n",
            "150\n",
            " of\n",
            " cases\n",
            " that\n",
            " have\n",
            " significant\n",
            " national\n",
            " importance\n",
            " or\n",
            " where\n",
            " lower\n",
            " courts\n",
            " disagree\n",
            " on\n",
            " federal\n",
            " law\n",
            ".\n",
            " \n",
            "\n",
            "\n",
            "-\n",
            " If\n",
            " 4\n",
            " out\n",
            " of\n",
            " the\n",
            " 9\n",
            " Just\n",
            "ices\n",
            " vote\n",
            " to\n",
            " grant\n",
            " cert\n",
            " (\n",
            "agree\n",
            " to\n",
            " hear\n",
            " the\n",
            " case\n",
            "),\n",
            " it\n",
            " goes\n",
            " on\n",
            " the\n",
            " Supreme\n",
            " Court\n",
            "'s\n",
            " do\n",
            "cket\n",
            " for\n",
            " arguments\n",
            ".\n",
            "\n",
            "\n",
            "-\n",
            " The\n",
            " Supreme\n",
            " Court\n",
            " then\n",
            " hears\n",
            " oral\n",
            " arguments\n",
            ",\n",
            " considers\n",
            " written\n",
            " brief\n",
            "s\n",
            ",\n",
            " examines\n",
            " the\n",
            " lower\n",
            " court\n",
            " records\n",
            ",\n",
            " and\n",
            " issues\n",
            " a\n",
            " final\n",
            " ruling\n",
            " on\n",
            " the\n",
            " case\n",
            ",\n",
            " which\n",
            " serves\n",
            " as\n",
            " binding\n",
            " precedent\n"
          ]
        }
      ],
      "source": [
        "# @title Streaming Example: Request Claude-2\n",
        "messages = [\n",
        "  {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
        "  {\"role\": \"user\", \"content\": \"how does a court case get to the Supreme Court?\"}\n",
        "  ]\n",
        "\n",
        "result = litellm.completion('claude-2', messages, stream=True)\n",
        "for part in result:\n",
        "    print(part.choices[0].delta.content or \"\")\n",
        "\n"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}