dist-fashion-mnist-fairing-.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 200215 14:42:02 config:123] Using preprocessor: <kubeflow.fairing.preprocessors.converted_notebook.ConvertNotebookPreprocessor object at 0x7f65b2937198>\n",
      "[I 200215 14:42:02 config:125] Using builder: <kubeflow.fairing.builders.append.append.AppendBuilder object at 0x7f65b2937748>\n",
      "[I 200215 14:42:02 config:127] Using deployer: <kubeflow.fairing.builders.append.append.AppendBuilder object at 0x7f65b2937748>\n",
      "[W 200215 14:42:02 append:50] Building image using Append builder...\n",
      "[I 200215 14:42:02 base:105] Creating docker context: /tmp/fairing_context_7iejmqld\n",
      "[I 200215 14:42:02 converted_notebook:127] Converting dist-fashion-mnist-fairing-.ipynb to dist-fashion-mnist-fairing-.py\n",
      "[I 200215 14:42:02 docker_creds_:234] Loading Docker credentials for repository 'brightfly/kubeflow-jupyter-lab:tf2.0-gpu'\n",
      "[W 200215 14:42:05 append:54] Image successfully built in 2.1764135679695755s.\n",
      "[W 200215 14:42:05 append:94] Pushing image kubeflow-registry.default.svc.cluster.local:30000/fairing-job:B0244C0F...\n",
      "[I 200215 14:42:05 docker_creds_:234] Loading Docker credentials for repository 'kubeflow-registry.default.svc.cluster.local:30000/fairing-job:B0244C0F'\n",
      "[W 200215 14:42:05 append:81] Uploading kubeflow-registry.default.svc.cluster.local:30000/fairing-job:B0244C0F\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:f6490e4900cb64767e994addad4a4329cca4ccfce37a63d21249d89b63ceb478 exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:9f0a21d58e5dce5512db6d5595c6e9c4ab014917cf0644e2d282b8f5e3f2522a exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:f9e7e7aaea7e58fd475e393007baf0251bad43cccaaef540a365ed1036173ae7 exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:d12137d65eaa8a740f8bc9419a0fe3e1ba963478fb9179142935e67d667c6f55 exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:7fb8ccf83688665b4137d132962349ffeae8bed7fee8f8b878fee4ecffe1f93d exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:251f5509d51d9e4119d4ffb70d4820f8e2d7dc72ad15df3ebd7cd755539e40fd exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:8810fcda1e6e2713f22a64b835ffa1ff15f49257f43dee869abef1929416d362 exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:06d09f7e28e650c4614bfeaa1336db8632aa02d48b9ff082d9cdf2edf4204376 exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:35c102085707f703de2d9eaad8752d6fe1b8f02b5d2149f1d8357c9cc7fb7d0a exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:d701a76e3193731210c61c838de0c3d8fdc8048b613ca88a58e11dc3223221ec exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:d1555fb0eeaac2bbfe413314cbbed0dea65a0a72312f29154e21afda1972b0d1 exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:180c1b9ae1a5a2a4aba2e08646155b250793be7a97a21556f2c4383f35510f14 exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:c4e2f5cde1e102db0e852d5e60ceac5bf377621b397ef3cb7bf1587052ea9520 exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:c73ad65f0763fc76b21c650ff9a5d7f4714c30a3dc27f462aa478c829a8efed1 exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:034db42ae0fa9a91805c16e78f4a6a9d8f5826c95c92713f835b2866333c786c exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:7d830fc21c34e37fa1d2f683b3392ce76489814b1973691f0593b2ce945b2aca exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:8e829fe70a46e3ac4334823560e98b257234c23629f19f05460e21a453091e6d exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:c592fdfb0e51000077def684bbdd097d6d870e62ae94418ba81a3468b03ec833 exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:0586a82d62e69da8dab24b635faf9c5da803ff9d208111a25b0e2253c97ec331 exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:6001e1789921cf851f6fb2e5fe05be70f482fe9c2286f66892fe5a3bc404569c exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:071fd910512489f1cf72f43901d730bfcf302bd46b88924e81f401810bd0e4ce exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:308e2d038b5334fb2ec9872230cfabd52b26102f6c8e0e257f3dcd63cc3b7177 exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:57418a6dc9496eeb6c9324e7bdf6fd48b9e1ac25cb473bf7b71f40d44e2f1e64 exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:65881bda3d6dd3c81420e22d3f921824b72ae8c07137aed41d02c640a2f87f04 exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:74a3420b0759a0426bc7a24daf48c0f376a7e8d293c015e0681884071c3ea0e1 exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:947e0f532378ce4f91ff44af563f21f5679d39a28efa2541594dd3f96730edb0 exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:b0a15f35c371773fb2237091edb340a15cf0e8615de8c9d8c4b8830c35a40265 exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:280] Layer sha256:db108ae5c97b9d9b4a6ab30f08b9f00de1a383adb8b2fbdad530e801661c12c5 exists, skipping\n",
      "[I 200215 14:42:05 docker_session_:284] Layer sha256:93382ba30d8427fe738de43854364d834268598422919f64cd3ffb4b97223d24 pushed.\n",
      "[I 200215 14:42:05 docker_session_:284] Layer sha256:fe4eb4c8edb6167de9da03bf5bfea361675851c5365a64ccbde697f2c071ef84 pushed.\n",
      "[I 200215 14:42:05 docker_session_:334] Finished upload of: kubeflow-registry.default.svc.cluster.local:30000/fairing-job:B0244C0F\n",
      "[W 200215 14:42:05 append:99] Pushed image kubeflow-registry.default.svc.cluster.local:30000/fairing-job:B0244C0F in 0.20229113392997533s.\n",
      "[W 200215 14:42:05 job:90] The job fairing-job-fdrp8 launched.\n",
      "[W 200215 14:42:05 manager:227] Waiting for fairing-job-fdrp8-hmn6r to start...\n",
      "[W 200215 14:42:05 manager:227] Waiting for fairing-job-fdrp8-hmn6r to start...\n",
      "[W 200215 14:42:05 manager:227] Waiting for fairing-job-fdrp8-hmn6r to start...\n",
      "[W 200215 14:42:06 manager:227] Waiting for fairing-job-fdrp8-hmn6r to start...\n",
      "[I 200215 14:42:07 manager:233] Pod started running True\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-labels-idx1-ubyte.gz\n",
      "32768/29515 [=================================] - 0s 0us/step\n",
      "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-images-idx3-ubyte.gz\n",
      "26427392/26421880 [==============================] - 0s 0us/step\n",
      "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-labels-idx1-ubyte.gz\n",
      "8192/5148 [===============================================] - 0s 0us/step\n",
      "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-images-idx3-ubyte.gz\n",
      "4423680/4422102 [==============================] - 0s 0us/step\n",
      "2020-02-15 14:42:10.267958: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1\n",
      "2020-02-15 14:42:10.317828: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:42:10.319094: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 0 with properties:\n",
      "name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235\n",
      "pciBusID: 0000:00:04.0\n",
      "2020-02-15 14:42:10.319261: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:42:10.320273: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 1 with properties:\n",
      "name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235\n",
      "pciBusID: 0000:00:05.0\n",
      "2020-02-15 14:42:10.320612: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0\n",
      "2020-02-15 14:42:10.322008: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0\n",
      "2020-02-15 14:42:10.323903: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10.0\n",
      "2020-02-15 14:42:10.324333: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10.0\n",
      "2020-02-15 14:42:10.326426: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10.0\n",
      "2020-02-15 14:42:10.328015: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10.0\n",
      "2020-02-15 14:42:10.332458: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7\n",
      "2020-02-15 14:42:10.332594: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:42:10.333641: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:42:10.334511: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:42:10.335356: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:42:10.336183: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Adding visible gpu devices: 0, 1\n",
      "2020-02-15 14:42:10.336802: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "2020-02-15 14:42:10.344355: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2200000000 Hz\n",
      "2020-02-15 14:42:10.345012: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x562ca10 executing computations on platform Host. Devices:\n",
      "2020-02-15 14:42:10.345059: I tensorflow/compiler/xla/service/service.cc:175]   StreamExecutor device (0): Host, Default Version\n",
      "2020-02-15 14:42:10.560200: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:42:10.562835: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:42:10.564156: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x56af280 executing computations on platform CUDA. Devices:\n",
      "2020-02-15 14:42:10.564187: I tensorflow/compiler/xla/service/service.cc:175]   StreamExecutor device (0): Tesla K80, Compute Capability 3.7\n",
      "2020-02-15 14:42:10.564197: I tensorflow/compiler/xla/service/service.cc:175]   StreamExecutor device (1): Tesla K80, Compute Capability 3.7\n",
      "2020-02-15 14:42:10.565077: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:42:10.566183: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 0 with properties:\n",
      "name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235\n",
      "pciBusID: 0000:00:04.0\n",
      "2020-02-15 14:42:10.566290: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:42:10.567313: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 1 with properties:\n",
      "name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235\n",
      "pciBusID: 0000:00:05.0\n",
      "2020-02-15 14:42:10.567371: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0\n",
      "2020-02-15 14:42:10.567385: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0\n",
      "2020-02-15 14:42:10.567403: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10.0\n",
      "2020-02-15 14:42:10.567414: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10.0\n",
      "2020-02-15 14:42:10.567431: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10.0\n",
      "2020-02-15 14:42:10.567441: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10.0\n",
      "2020-02-15 14:42:10.567456: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7\n",
      "2020-02-15 14:42:10.567555: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:42:10.568561: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:42:10.569901: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:42:10.571137: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:42:10.572070: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Adding visible gpu devices: 0, 1\n",
      "2020-02-15 14:42:10.572172: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0\n",
      "2020-02-15 14:42:10.574722: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1159] Device interconnect StreamExecutor with strength 1 edge matrix:\n",
      "2020-02-15 14:42:10.574754: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1165]      0 1\n",
      "2020-02-15 14:42:10.574766: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1178] 0:   N Y\n",
      "2020-02-15 14:42:10.574774: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1178] 1:   Y N\n",
      "2020-02-15 14:42:10.574937: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:42:10.575932: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:42:10.577146: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:42:10.578199: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1304] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10804 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)\n",
      "2020-02-15 14:42:10.578814: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:42:10.579835: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1304] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:1 with 10804 MB memory) -> physical GPU (device: 1, name: Tesla K80, pci bus id: 0000:00:05.0, compute capability: 3.7)\n",
      "장치의 수: 2\n",
      "Traceback (most recent call last):\n",
      "  File \"/app/dist-fashion-mnist-fairing-.py\", line 58, in <module>\n",
      "    remote_train.train()\n",
      "  File \"/app/dist-fashion-mnist-fairing-.py\", line 24, in train\n",
      "    tf.keras.layers.Dense(10, activation='softmax')\n",
      "  File \"/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/training/tracking/base.py\", line 457, in _method_wrapper\n",
      "    result = method(self, *args, **kwargs)\n",
      "  File \"/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/sequential.py\", line 114, in __init__\n",
      "    self.add(layer)\n",
      "  File \"/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/training/tracking/base.py\", line 457, in _method_wrapper\n",
      "    result = method(self, *args, **kwargs)\n",
      "  File \"/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/sequential.py\", line 196, in add\n",
      "    output_tensor = layer(self.outputs[0])\n",
      "  File \"/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/base_layer.py\", line 817, in __call__\n",
      "    self._maybe_build(inputs)\n",
      "  File \"/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/base_layer.py\", line 2141, in _maybe_build\n",
      "    self.build(input_shapes)\n",
      "  File \"/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/layers/core.py\", line 1027, in build\n",
      "    trainable=True)\n",
      "  File \"/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/base_layer.py\", line 2311, in __setattr__\n",
      "    if val.trainable:\n",
      "  File \"/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/variables.py\", line 477, in trainable\n",
      "    raise NotImplementedError\n",
      "NotImplementedError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[W 200215 14:42:12 job:162] Cleaning up job fairing-job-fdrp8...\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import os\n",
    "\n",
    "class MyFashionMnist(object):\n",
    "  def train(self):\n",
    "  \n",
    "    (x_train, y_train), (x_test, y_test) = tf.keras.datasets.fashion_mnist.load_data()\n",
    "    \n",
    "    strategy = tf.distribute.experimental.CentralStorageStrategy()\n",
    "    print('장치의 수: {}'.format(strategy.num_replicas_in_sync))      \n",
    "    x_train, x_test = x_train / 255.0, x_test / 255.0\n",
    "\n",
    "    with strategy.scope():    \n",
    "        model = tf.keras.models.Sequential([\n",
    "          tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
    "          tf.keras.layers.Dense(128, activation='relu'),\n",
    "          tf.keras.layers.Dropout(0.2),\n",
    "          tf.keras.layers.Dense(10, activation='softmax')\n",
    "        ])\n",
    "        model.summary()\n",
    "        model.compile(optimizer='adam',\n",
    "                      loss='sparse_categorical_crossentropy',\n",
    "                      metrics=['accuracy'])\n",
    "\n",
    "        model.fit(x_train, y_train, epochs=10)\n",
    "\n",
    "        model.evaluate(x_test,  y_test, verbose=2)\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    if os.getenv('FAIRING_RUNTIME', None) is None:\n",
    "        from kubeflow import fairing\n",
    "        from kubeflow.fairing.kubernetes import utils as k8s_utils\n",
    "\n",
    "        DOCKER_REGISTRY = 'kubeflow-registry.default.svc.cluster.local:30000'\n",
    "        fairing.config.set_builder(\n",
    "            'append',\n",
    "            image_name='fairing-job',\n",
    "            base_image='brightfly/kubeflow-jupyter-lab:tf2.0-gpu',\n",
    "            registry=DOCKER_REGISTRY, \n",
    "            push=True)\n",
    "        # cpu 2, memory 5GiB\n",
    "        fairing.config.set_deployer('job',\n",
    "                                    namespace='dudaji',\n",
    "                                    pod_spec_mutators=[\n",
    "                                        k8s_utils.get_resource_mutator(cpu=2,\n",
    "                                                                       memory=5)]\n",
    "         \n",
    "                                   )\n",
    "        fairing.config.run()\n",
    "    else:\n",
    "        remote_train = MyFashionMnist()\n",
    "        remote_train.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}