1.mlops.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "28d1dcad-2ae3-4af0-9dde-654654d0da34",
   "metadata": {},
   "source": [
    "## AutoReload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2fca3fcb-741b-447d-b639-68fac96e7656",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8112809d-8e35-4955-aaf5-047e447abf4e",
   "metadata": {},
   "source": [
    "## 0. parameter store"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6753b726-eede-4202-9bca-7bca804de5de",
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "from utils.ssm import parameter_store"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "484c8732-455d-47c9-8aa8-d3c659d55301",
   "metadata": {},
   "outputs": [],
   "source": [
    "strRegionName=boto3.Session().region_name\n",
    "pm = parameter_store(strRegionName)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f522683-2aa1-4c28-98bb-e42056cc4b9a",
   "metadata": {},
   "source": [
    "## 1. Dataset for training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "164b5940-7311-4fad-9608-9789653a1fa7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "04fcdd04-4d34-44f3-90fd-e1ee02fa5d5b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'s3://sagemaker-ap-northeast-2-419974056037/reviews-helpfulness-pipeline/data/reviews.tsv.gz'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strSrcPath = \"s3://amazon-reviews-pds/tsv/amazon_reviews_us_Electronics_v1_00.tsv.gz\"\n",
    "strTargetPath = f\"s3://{pm.get_params(key='DAT-BUCKET')}/reviews-helpfulness-pipeline/data/reviews.tsv.gz\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "db64934d-a640-4286-8933-80ae8b284b96",
   "metadata": {},
   "outputs": [],
   "source": [
    "strQuery = \"\".join([\"aws s3 cp \", strSrcPath, \" \", strTargetPath])\n",
    "strResponse = os.popen(strQuery).read()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "216c334e-cc60-4d23-8ed1-9da22321472d",
   "metadata": {},
   "source": [
    "### Delete bucket (if needed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "1e3c9ae2-76c3-4613-8e5c-77275d00d7cb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'sagemaker-ap-northeast-2-419974056037'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strBucketName = pm.get_params(key=\"DAT-BUCKET\")\n",
    "strBucketName"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "20d482a4-684c-4433-8e8b-1019ee3153d4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "strQuery = \"\".join([\"aws s3 rm --recursive \", '\"s3://', strBucketName, '\"'])\n",
    "strResponse = os.popen(strQuery).read()\n",
    "strQuery = \"\".join([\"aws s3 rb \", '\"s3://', strBucketName, '\"'])\n",
    "strResponse = os.popen(strQuery).read()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a3bd85c-87c6-4d2c-b230-b8ed3a5d2f2e",
   "metadata": {},
   "source": [
    "## 2. Build a docker and Register the image to ECR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2edfb16a-8499-4885-b7ea-9c4172cac181",
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.ecr import ecr_handler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "799da6a5-57c7-40a7-8da7-bedd68bcd8d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "ecr = ecr_handler()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "39852f78-a98f-4b7c-aa64-b7aaea77c3f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "strRepositoryName = \"dat-processing-container\"\n",
    "strDockerDir = \"./docker/\"\n",
    "strTag = \":latest\"\n",
    "strRegionName=pm.get_params(key=\"DAT-REGION\")\n",
    "strAccountId=pm.get_params(key=\"DAT-ACCOUNT-ID\")    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "3e9dff7a-d377-4210-93b7-39e3d616a8c8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/ec2-user/SageMaker/dynamic-ab-testing\n",
      "/home/ec2-user/SageMaker/dynamic-ab-testing\n"
     ]
    }
   ],
   "source": [
    "ecr.build_docker(strDockerDir, strRepositoryName)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "82d44ddc-23fc-4bd2-b5d5-0f68ca92cf31",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "== REGISTER AN IMAGE TO ECR ==\n",
      "  processing_repository_uri: 419974056037.dkr.ecr.ap-northeast-2.amazonaws.com/dat-processing-container:latest\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING! Using --password via the CLI is insecure. Use --password-stdin.\n",
      "WARNING! Your password will be stored unencrypted in /home/ec2-user/.docker/config.json.\n",
      "Configure a credential helper to remove this warning. See\n",
      "https://docs.docker.com/engine/reference/commandline/login/#credentials-store\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "== REGISTER AN IMAGE TO ECR ==\n",
      "==============================\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'Store suceess'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strEcrRepositoryUri = ecr.register_image_to_ecr(strRegionName, strAccountId, strRepositoryName, strTag)\n",
    "pm.put_params(key=\"DAT-PROCESSING-ECR-URI\", value=strEcrRepositoryUri, overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5e769c5-7223-42f5-b83c-df9b0c658993",
   "metadata": {},
   "source": [
    "## 3. Set code-pipeLine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "cea49216-830b-41ef-ac89-d93e506fa4f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import boto3\n",
    "import sagemaker\n",
    "from os import path\n",
    "from utils.code_pipeline import code_pipeline_handler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "be1a6fae-9173-4419-b35e-c12ca4736672",
   "metadata": {},
   "outputs": [],
   "source": [
    "cph = code_pipeline_handler()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9346e3ba-5d3a-414f-96dd-0d44bca1c765",
   "metadata": {},
   "source": [
    "#### -- args for codecommit, build and pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "0175e21f-b04a-4f91-9461-58774bb6d544",
   "metadata": {},
   "outputs": [],
   "source": [
    "strRepoName = \"mlops\"\n",
    "strReopDesc = \"MLOps for Dynamic A/B Testing\"\n",
    "strSrcDir = os.path.join(os.getcwd(), \"./mlops/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c20756f-c742-40b8-8f0b-fb4b0e9f2377",
   "metadata": {},
   "source": [
    "### 3.1 code-commit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "18fb3dfb-c849-4154-b02a-c1706c64c08a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "== CREATE REPO ==\n",
      "  Repository name [mlops] was successfully created!!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Cloning into 'tmp'...\n",
      "warning: You appear to have cloned an empty repository.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "/home/ec2-user/SageMaker/dynamic-ab-testing\n",
      "/home/ec2-user/SageMaker/dynamic-ab-testing/mlops\n",
      "\n",
      "[master (root-commit) 9fc3a6b] Initial commit\n",
      " Committer: EC2 Default User <ec2-user@ip-172-16-95-57.ap-northeast-2.compute.internal>\n",
      "Your name and email address were configured automatically based\n",
      "on your username and hostname. Please check that they are accurate.\n",
      "You can suppress this message by setting them explicitly:\n",
      "\n",
      "    git config --global user.name \"Your Name\"\n",
      "    git config --global user.email you@example.com\n",
      "\n",
      "After doing this, you may fix the identity used for this commit with:\n",
      "\n",
      "    git commit --amend --reset-author\n",
      "\n",
      " 23 files changed, 1984 insertions(+)\n",
      " create mode 100644 .ipynb_checkpoints/codebuild-buildspec-checkpoint.yml\n",
      " create mode 100644 codebuild-buildspec.yml\n",
      " create mode 100644 pipeline/.ipynb_checkpoints/mlops_pipeline-checkpoint.py\n",
      " create mode 100644 pipeline/config/.ipynb_checkpoints/config-checkpoint.ini\n",
      " create mode 100644 pipeline/config/.ipynb_checkpoints/config-checkpoint.py\n",
      " create mode 100644 pipeline/config/__init__.py\n",
      " create mode 100644 pipeline/config/__pycache__/__init__.cpython-37.pyc\n",
      " create mode 100644 pipeline/config/__pycache__/config.cpython-37.pyc\n",
      " create mode 100644 pipeline/config/config.ini\n",
      " create mode 100644 pipeline/config/config.py\n",
      " create mode 100644 pipeline/mlops_pipeline.py\n",
      " create mode 100644 pipeline/utils/.ipynb_checkpoints/ssm-checkpoint.py\n",
      " create mode 100644 pipeline/utils/__init__.py\n",
      " create mode 100644 pipeline/utils/__pycache__/__init__.cpython-37.pyc\n",
      " create mode 100644 pipeline/utils/__pycache__/ssm.cpython-37.pyc\n",
      " create mode 100644 pipeline/utils/ssm.py\n",
      " create mode 100644 source/approval/.ipynb_checkpoints/approval-checkpoint.py\n",
      " create mode 100644 source/approval/approval.py\n",
      " create mode 100644 source/deploy/deploy.py\n",
      " create mode 100644 source/evaluation/.ipynb_checkpoints/evaluation-checkpoint.py\n",
      " create mode 100644 source/evaluation/evaluation.py\n",
      " create mode 100644 source/preprocessing/.ipynb_checkpoints/preprocessing-checkpoint.py\n",
      " create mode 100644 source/preprocessing/preprocessing.py\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "To https://git-codecommit.ap-northeast-2.amazonaws.com/v1/repos/mlops\n",
      " * [new branch]      master -> master\n"
     ]
    }
   ],
   "source": [
    "strCloneURL = cph.create_repository(strRepoName, strReopDesc)\n",
    "cph.clone_from_url(strSrcDir, strCloneURL)\n",
    "cph.initial_commit_push(strSrcDir)\n",
    "#cph.delete_repository(strRepoName)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0aee8ed-8138-4999-8bad-63996f9efdde",
   "metadata": {},
   "source": [
    "### 3.2 CodeBuild"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "2ef7f9ee-d5fa-4314-8e71-176cd5b4d8e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "strCodeBuildPJTName=\"mlops-build\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "4235f9a1-33fb-4939-8a5a-1771d2f5cca0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Argments for CodeBuild below:\n",
      "{'artifacts': {'name': 'mlops-build', 'type': 'CODEPIPELINE'},\n",
      " 'badgeEnabled': False,\n",
      " 'cache': {'type': 'NO_CACHE'},\n",
      " 'description': 'Hello Build Project',\n",
      " 'environment': {'computeType': 'BUILD_GENERAL1_SMALL',\n",
      "                 'environmentVariables': [{'name': 'IMAGE_REPO_NAME',\n",
      "                                           'type': 'PLAINTEXT',\n",
      "                                           'value': 'iris-model'},\n",
      "                                          {'name': 'IMAGE_TAG',\n",
      "                                           'type': 'PLAINTEXT',\n",
      "                                           'value': 'latest'},\n",
      "                                          {'name': 'AWS_ACCOUNT_ID',\n",
      "                                           'type': 'PLAINTEXT',\n",
      "                                           'value': '419974056037'},\n",
      "                                          {'name': 'AWS_DEFAULT_REGION',\n",
      "                                           'type': 'PLAINTEXT',\n",
      "                                           'value': 'ap-northeast-2'},\n",
      "                                          {'name': 'TEMPLATE_BUCKET',\n",
      "                                           'type': 'PLAINTEXT',\n",
      "                                           'value': 'sagemaker-ap-northeast-2-419974056037'},\n",
      "                                          {'name': 'TEMPLATE_PREFIX',\n",
      "                                           'type': 'PLAINTEXT',\n",
      "                                           'value': 'codebuild'}],\n",
      "                 'image': 'aws/codebuild/amazonlinux2-x86_64-standard:3.0',\n",
      "                 'imagePullCredentialsType': 'CODEBUILD',\n",
      "                 'privilegedMode': False,\n",
      "                 'type': 'LINUX_CONTAINER'},\n",
      " 'logsConfig': {'cloudWatchLogs': {'status': 'ENABLED'},\n",
      "                's3Logs': {'encryptionDisabled': False, 'status': 'DISABLED'}},\n",
      " 'name': 'mlops-build',\n",
      " 'queuedTimeoutInMinutes': 480,\n",
      " 'serviceRole': 'arn:aws:iam::419974056037:role/DAT-CodeBuildRole',\n",
      " 'source': {'buildspec': 'codebuild-buildspec.yml', 'type': 'CODEPIPELINE'},\n",
      " 'timeoutInMinutes': 60}\n"
     ]
    }
   ],
   "source": [
    "cph.create_build_project(\n",
    "    strCodeBuildPJTName=strCodeBuildPJTName,\n",
    "    strAccountId=pm.get_params(key=\"DAT-ACCOUNT-ID\"),\n",
    "    strBuildServiceRoleARN=pm.get_params(key=\"DAT-CODE-BUILD-ROLE-ARN\"),\n",
    "    strRegionName=pm.get_params(key=\"DAT-REGION\"),\n",
    "    strBucketName=pm.get_params(key=\"DAT-BUCKET\"),\n",
    ")\n",
    "#!aws codebuild delete-project --name \"mlops-build\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f9fa704e-410f-4762-be24-2c99ba61924b",
   "metadata": {},
   "source": [
    "### 3.3. CodePipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c1c7083c-8b9c-4437-ae41-5f0a0e9a86cc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'arn:aws:iam::419974056037:role/DAT-CodePipelineRole'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pm.get_params(key=\"DAT-CODE-PIPELINE-ROLE-ARN\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "a853ce2c-fe16-4c4b-a593-f9e8e5c04650",
   "metadata": {},
   "outputs": [],
   "source": [
    "strCodePipelineName = \"mlops-code-pipeline\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "983457cd-8f27-42eb-ba34-74d6935c2ce9",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Couldn't find Pipeline: [mlops-code-pipeline], so, create new pipeline.\n",
      "Create CodePipeline\n",
      "Argments for CodeBuild below:\n",
      "{'pipeline': {'artifactStore': {'location': 'sagemaker-ap-northeast-2-419974056037',\n",
      "                                'type': 'S3'},\n",
      "              'name': 'mlops-code-pipeline',\n",
      "              'roleArn': 'arn:aws:iam::419974056037:role/DAT-CodePipelineRole',\n",
      "              'stages': [{'actions': [{'actionTypeId': {'category': 'Source',\n",
      "                                                        'owner': 'AWS',\n",
      "                                                        'provider': 'CodeCommit',\n",
      "                                                        'version': '1'},\n",
      "                                       'configuration': {'BranchName': 'master',\n",
      "                                                         'OutputArtifactFormat': 'CODE_ZIP',\n",
      "                                                         'PollForSourceChanges': 'false',\n",
      "                                                         'RepositoryName': 'mlops'},\n",
      "                                       'inputArtifacts': [],\n",
      "                                       'name': 'Source',\n",
      "                                       'namespace': 'SourceVariables',\n",
      "                                       'outputArtifacts': [{'name': 'SourceArtifact'}],\n",
      "                                       'region': 'ap-northeast-2',\n",
      "                                       'runOrder': 1}],\n",
      "                          'name': 'Source'},\n",
      "                         {'actions': [{'actionTypeId': {'category': 'Build',\n",
      "                                                        'owner': 'AWS',\n",
      "                                                        'provider': 'CodeBuild',\n",
      "                                                        'version': '1'},\n",
      "                                       'configuration': {'ProjectName': 'mlops-build'},\n",
      "                                       'inputArtifacts': [{'name': 'SourceArtifact'}],\n",
      "                                       'name': 'BuildAndExecuteSageMakerPipeline',\n",
      "                                       'namespace': 'BuildVariables',\n",
      "                                       'outputArtifacts': [{'name': 'BuildArtifact'}],\n",
      "                                       'region': 'ap-northeast-2',\n",
      "                                       'runOrder': 1}],\n",
      "                          'name': 'Build'}],\n",
      "              'version': 1}}\n"
     ]
    }
   ],
   "source": [
    "cph = code_pipeline_handler()\n",
    "cph.create_execute_code_pipeline(\n",
    "    strCodePipelineName=strCodePipelineName,\n",
    "    strPipelineRoleARN=pm.get_params(key=\"DAT-CODE-PIPELINE-ROLE-ARN\"),\n",
    "    strRegionName=pm.get_params(key=\"DAT-REGION\"),\n",
    "    strBucketName=pm.get_params(key=\"DAT-BUCKET\"),\n",
    "    strRepoName=strRepoName,\n",
    "    strCodeBuildPJTName=strCodeBuildPJTName,\n",
    ")\n",
    "#!aws codepipeline delete-pipeline --name \"mlops-code-pipeline\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "472183da-7013-492d-9a5a-d0cbf55d3d24",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start pipeline execution for [mlops-code-pipeline]\n"
     ]
    }
   ],
   "source": [
    "## Start pipeline\n",
    "cph.start_pipeline_execution(strCodePipelineName)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06792071-bf93-4875-a224-1f207ecab168",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}