Staging/migrate iplanrio flows

.github/workflows/cd.yaml

@@ -38,7 +38,7 @@ jobs:
 
       - name: Install Python dependencies for deploying
         run: |-
-          pip install -U pip poetry
+          pip install -U pip "poetry<1.8.0"
           poetry config virtualenvs.create false
           poetry install --with dev --with ci
 

.github/workflows/cd_staging.yaml

@@ -40,7 +40,7 @@ jobs:
 
       - name: Install Python dependencies for deploying
         run: |-
-          pip install -U pip poetry
+          pip install -U pip "poetry<1.8.0"
           poetry config virtualenvs.create false
           poetry install --with dev --with ci
 

.github/workflows/code-tree-analysis.yaml

@@ -17,7 +17,7 @@ jobs:
 
       - name: Install Python dependencies for deploying
         run: |-
-          pip install -U pip poetry
+          pip install -U pip "poetry<1.8.0"
           poetry config virtualenvs.create false
           poetry install --with dev --with ci
 

.github/workflows/lint.yaml

@@ -15,6 +15,7 @@ jobs:
         uses: hadolint/hadolint-action@54c9adbab1582c2ef04b2016b760714a4bfde3cf
         with:
           dockerfile: Dockerfile
+          ignore: DL3008
 
       - name: Set up Python
         uses: actions/setup-python@e9aba2c848f5ebd159c070c61ea2c4e2b122355e
@@ -23,7 +24,7 @@ jobs:
 
       - name: Set up Poetry and upgrade pip
         run: |
-          pip install -U pip poetry
+          pip install -U pip "poetry<1.8.0"
 
       - name: Install dependencies
         run: |

Dockerfile

@@ -1,19 +1,46 @@
 # Build arguments
 ARG PYTHON_VERSION=3.10-slim
 
+# Get Oracle Instant Client
+FROM curlimages/curl:7.81.0 as curl-step
+ARG ORACLE_INSTANT_CLIENT_URL=https://download.oracle.com/otn_software/linux/instantclient/215000/instantclient-basic-linux.x64-21.5.0.0.0dbru.zip
+RUN curl -sSLo /tmp/instantclient.zip $ORACLE_INSTANT_CLIENT_URL
+
+# Unzip Oracle Instant Client
+FROM ubuntu:18.04 as unzip-step
+COPY --from=curl-step /tmp/instantclient.zip /tmp/instantclient.zip
+RUN apt-get update && \
+    apt-get install --no-install-recommends -y unzip && \
+    rm -rf /var/lib/apt/lists/* && \
+    unzip /tmp/instantclient.zip -d /tmp
+
 # Start Python image
 FROM python:${PYTHON_VERSION}
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 # Install git
 RUN apt-get update && \
-    apt-get install -y git && \
+    apt-get install -y --no-install-recommends git && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
 # Setting environment with prefect version
 ARG PREFECT_VERSION=1.4.1
 ENV PREFECT_VERSION $PREFECT_VERSION
 
+# Setup Oracle Instant Client and SQL Server ODBC Driver
+WORKDIR /opt/oracle
+COPY --from=unzip-step /tmp/instantclient_21_5 /opt/oracle/instantclient_21_5
+RUN apt-get update && \
+    apt-get install --no-install-recommends -y curl gnupg2 libaio1 && \
+    curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \
+    echo "deb [arch=amd64,arm64,armhf] https://packages.microsoft.com/debian/12/prod bookworm main" > /etc/apt/sources.list.d/mssql-release.list && \
+    apt-get update && \
+    ACCEPT_EULA=Y apt-get install --no-install-recommends -y ffmpeg libsm6 libxext6 msodbcsql17 openssl unixodbc-dev && \
+    rm -rf /var/lib/apt/lists/* && \
+    sh -c "echo /opt/oracle/instantclient_21_5 > /etc/ld.so.conf.d/oracle-instantclient.conf" && \
+    ldconfig
+
 # Setup virtual environment and prefect
 ENV VIRTUAL_ENV=/opt/venv
 RUN python3 -m venv $VIRTUAL_ENV
@@ -23,4 +50,4 @@ RUN python3 -m pip install --no-cache-dir -U "pip>=21.2.4" "prefect==$PREFECT_VE
 # Install requirements
 WORKDIR /app
 COPY . .
-RUN python3 -m pip install --prefer-binary --no-cache-dir -U .
+RUN python3 -m pip install --prefer-binary --no-cache-dir -U .

pipelines/constants.py

@@ -11,12 +11,13 @@ class constants(Enum):
     DOCKER_TAG = "AUTO_REPLACE_DOCKER_TAG"
     DOCKER_IMAGE_NAME = "AUTO_REPLACE_DOCKER_IMAGE"
     DOCKER_IMAGE = f"{DOCKER_IMAGE_NAME}:{DOCKER_TAG}"
-    GCS_FLOWS_BUCKET = "rj-orgao-flows"
+    GCS_FLOWS_BUCKET = "datario-public"
 
     ######################################
     # Agent labels
     ######################################
     # EXAMPLE_AGENT_LABEL = "example_agent"
+    RJ_IPLANRIO_AGENT_LABEL = "iplanrio"
 
     ######################################
     # Other constants

pipelines/flows.py

@@ -2,4 +2,6 @@
 """
 Imports all flows for every project so we can register all of them.
 """
-from pipelines.exemplo import *  # noqa
+from pipelines.processorio import *  # noqa
+from pipelines.sicop import *  # noqa
+from pipelines.templates import *  # noqa

pipelines/processorio/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+
+from pipelines.processorio.dump_db.flows import *  # noqa

pipelines/processorio/dump_db/flows.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+"""
+Database dumping flows for processorio.
+"""
+
+
+from copy import deepcopy
+
+from prefect.run_configs import KubernetesRun
+from prefect.storage import GCS
+from prefeitura_rio.pipelines_templates.dump_db.flows import flow as dump_sql_flow
+from prefeitura_rio.pipelines_utils.prefect import set_default_parameters
+from prefeitura_rio.pipelines_utils.state_handlers import (
+    handler_initialize_sentry,
+    handler_inject_bd_credentials,
+)
+
+from pipelines.constants import constants
+from pipelines.processorio.dump_db.schedules import (
+    processorio_infra_daily_update_schedule,
+)
+
+rj_iplanrio_processorio_flow = deepcopy(dump_sql_flow)
+rj_iplanrio_processorio_flow.state_handlers = [
+    handler_inject_bd_credentials,
+    handler_initialize_sentry,
+]
+rj_iplanrio_processorio_flow.name = "IPLANRIO: processo.rio - Ingerir tabelas de banco SQL"
+rj_iplanrio_processorio_flow.storage = GCS(constants.GCS_FLOWS_BUCKET.value)
+
+rj_iplanrio_processorio_flow.run_config = KubernetesRun(
+    image=constants.DOCKER_IMAGE.value,
+    labels=[
+        constants.RJ_IPLANRIO_AGENT_LABEL.value,  # label do agente
+    ],
+)
+
+processorio_default_parameters = {
+    "db_database": "SIGADOC.PCRJ",
+    "db_host": "10.70.6.64",
+    "db_port": "1521",
+    "db_type": "oracle",
+    "dataset_id": "administracao_servicos_publicos",
+    "vault_secret_path": "processorio-prod",
+}
+
+rj_iplanrio_processorio_flow = set_default_parameters(
+    rj_iplanrio_processorio_flow,
+    default_parameters=processorio_default_parameters,
+)
+
+rj_iplanrio_processorio_flow.schedule = processorio_infra_daily_update_schedule