Mini_ETL_Tratamento_Exportacao.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Mini_ETL_Tratamento_Exportacao.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyO5yKSfnv4JBIAHIu2/G+X7",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/naokityokoyama/Projetos_Diversos/blob/master/Mini_ETL_Tratamento_Exportacao.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_vMJ3BWm96NC"
      },
      "source": [
        "import numpy as np\r\n",
        "import re\r\n",
        "import csv\r\n",
        "import os\r\n",
        "import pandas as pd\r\n",
        "from dateutil import parser\r\n",
        "from datetime import datetime\r\n",
        "path = '/tmp/DET/'\r\n",
        "\r\n",
        "#funcao importa \r\n",
        "def importa(link):\r\n",
        "  arquivo = open(link)\r\n",
        "  linhas = csv.reader(arquivo)\r\n",
        "  lista_csv = []\r\n",
        "  for linha in linhas:\r\n",
        "    lista_csv.append(linha)\r\n",
        "  return lista_csv   \r\n",
        "\r\n",
        "#funcao regex\r\n",
        "def rege(importado):\r\n",
        "  resultado1 = []\r\n",
        "  for i in range(1, len(importado)):\r\n",
        "    resultado2 = []\r\n",
        "    resultado1.append(re.sub(r'$', '.000000Z', importado[i][0]))\r\n",
        "    for j in range(len(resultado1)):\r\n",
        "      resultado2.append(re.sub(r' ', 'T', resultado1[j]))\r\n",
        "  return resultado2\r\n",
        "\r\n",
        "while True:\r\n",
        " for file in os.listdir(path):\r\n",
        "   if file.startswith(\"DET_Export_LOL-SLA_Traceroute_Active\"):\r\n",
        "    csvi = (os.path.join(path, file))\r\n",
        "  \r\n",
        "    importado = importa(csvi)\r\n",
        "    data_tratada = rege(importado)\r\n",
        "  \r\n",
        "    #replace\r\n",
        "    for i,j in zip(range(1,len(importado)), range(len(data_tratada))):\r\n",
        "      importado[i][0]= importado[i][0].replace(importado[i][0], data_tratada[j])\r\n",
        "  \r\n",
        "    #join \r\n",
        "    final = []\r\n",
        "    for i in importado:\r\n",
        "      final.append(','.join(i))\r\n",
        "    final\r\n",
        "    #head\r\n",
        "    st1 = '#group false,true,true,true,true,true,true,false,false,false,false,true,false,false,false,false,false,true,true,false'\r\n",
        "    st2 = '#datatype time,tag,tag,tag,tag,measurement,tag,long,long,long,long,tag,long,long,long,long,long,tag,tag,string'\r\n",
        "    st3 = \"Time_Stamp,Test_Instance_ID,Verifier,SLA_ID,SLA_Name,Service_Instance_Name,Interface_Category,Result,Hop_Count_to_Destination,Number_of_ICMP_Replies_From_Destination,Network_Topology_Change,Destination_IP_Address,Minimum_Round-Trip_Latency_to_Destination,Maximum_Round-Trip_Latency_to_Destination,Median_Round-Trip_Latency_to_Destination,Average_Round-Trip_Latency_to_Destination,Standard_Deviation,Host_Tested,Source_Interface_Address,Route_Hash\"\r\n",
        "\r\n",
        "    final.insert(0, st1)\r\n",
        "    final.insert(0, st2)\r\n",
        "    final[2]=st3\r\n",
        "\r\n",
        "    np.savetxt(\"final.csv\", final, fmt='%s')\r\n",
        "\r\n",
        "    influx = 'influx write -b LoL -f final.csv'\r\n",
        "    os.system(influx)\r\n",
        "    copia = 'cp '+ csvi + ' /tmp/DET/processado/'\r\n",
        "    apaga = 'rm '+ csvi\r\n",
        "    os.system(copia)\r\n",
        "    os.system(apaga)\r\n"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}