tokenizer_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9dd6b902",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import AutoTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fdd5c0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"distilbert-base-cased\") # \"distilbert-base-cased\", \"xlm-roberta-base\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a02a547",
   "metadata": {},
   "outputs": [],
   "source": [
    "words = [\"EU\", \"rejects\", \"German\", \"call\", \"to\", \"boycott\", \"British\", \"lamb\", \".\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "587b4c8c",
   "metadata": {},
   "outputs": [],
   "source": [
    "tok_input = tokenizer(words, is_split_into_words = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "205e7e6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "tok_input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "304d3a79",
   "metadata": {},
   "outputs": [],
   "source": [
    "tok_input.word_ids(batch_index = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f80d876",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens = tokenizer.convert_ids_to_tokens(tok_input['input_ids'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f21c2f0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35e24953",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0569755e",
   "metadata": {},
   "outputs": [],
   "source": [
    "a = torch.tensor([[1, 2], [3, 4]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbfdb5e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88d591d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "a.view(-1, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b47b7de4",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.flatten(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "849b242c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}