diff --git a/.github/ISSUE_TEMPLATE/notifica--o-de-problema.md b/.github/ISSUE_TEMPLATE/notifica--o-de-problema.md index acc81b67..9294f8f0 100644 --- a/.github/ISSUE_TEMPLATE/notifica--o-de-problema.md +++ b/.github/ISSUE_TEMPLATE/notifica--o-de-problema.md @@ -3,7 +3,7 @@ name: Notificação de problema about: Notifique um problema que está ocorrendo para que possamos corrigí-lo. title: "[BUG]" labels: bug -assignees: +assignees: --- **Descreva o problema** diff --git a/.github/ISSUE_TEMPLATE/requerimento-de-funcionalidade.md b/.github/ISSUE_TEMPLATE/requerimento-de-funcionalidade.md index 347d97f1..51d99197 100644 --- a/.github/ISSUE_TEMPLATE/requerimento-de-funcionalidade.md +++ b/.github/ISSUE_TEMPLATE/requerimento-de-funcionalidade.md @@ -3,7 +3,7 @@ name: Requerimento de funcionalidade about: Sugira novas funcionalidades que podem ser úteis a todos os desenvolvedores. title: "[FEATURE]" labels: enhancement -assignees: +assignees: --- **Essa funcionalidade está relacionada a um problema?** diff --git a/.github/workflows/scripts/register_flows.py b/.github/workflows/scripts/register_flows.py index 71af7edf..eaf568f6 100644 --- a/.github/workflows/scripts/register_flows.py +++ b/.github/workflows/scripts/register_flows.py @@ -91,11 +91,7 @@ def build_and_register( # pylint: disable=too-many-branches while attempts < max_retries: attempts += 1 try: - ( - flow_id, - flow_version, - is_new, - ) = register_serialized_flow( + (flow_id, flow_version, is_new,) = register_serialized_flow( client=client, serialized_flow=serialized_flow, project_id=project_id, diff --git a/pipelines/meteorologia/meteorologia_inmet/flows.py b/pipelines/meteorologia/meteorologia_inmet/flows.py index 90d4fd29..982639aa 100644 --- a/pipelines/meteorologia/meteorologia_inmet/flows.py +++ b/pipelines/meteorologia/meteorologia_inmet/flows.py @@ -5,21 +5,20 @@ """ from datetime import timedelta -from prefect import case, Parameter +from prefect import Parameter, case from prefect.run_configs import KubernetesRun from prefect.storage import GCS from prefect.tasks.prefect import create_flow_run, wait_for_flow_run from pipelines.constants import constants -from pipelines.utils.constants import constants as utils_constants -from pipelines.meteorologia.meteorologia_inmet.tasks import ( - get_dates, - # slice_data, +from pipelines.meteorologia.meteorologia_inmet.schedules import hour_schedule +from pipelines.meteorologia.meteorologia_inmet.tasks import ( # slice_data, download, - tratar_dados, + get_dates, salvar_dados, + tratar_dados, ) -from pipelines.meteorologia.meteorologia_inmet.schedules import hour_schedule +from pipelines.utils.constants import constants as utils_constants from pipelines.utils.decorators import Flow from pipelines.utils.dump_db.constants import constants as dump_db_constants from pipelines.utils.dump_to_gcs.constants import constants as dump_to_gcs_constants @@ -28,7 +27,6 @@ get_current_flow_labels, ) - with Flow( name="COR: Meteorologia - Meteorologia INMET", code_owners=[ diff --git a/pipelines/meteorologia/meteorologia_inmet/schedules.py b/pipelines/meteorologia/meteorologia_inmet/schedules.py index 8030e9e8..ddecdb4c 100644 --- a/pipelines/meteorologia/meteorologia_inmet/schedules.py +++ b/pipelines/meteorologia/meteorologia_inmet/schedules.py @@ -3,9 +3,11 @@ Schedules for meteorologia_inmet Rodar a cada 1 hora """ -from datetime import timedelta, datetime +from datetime import datetime, timedelta + from prefect.schedules import Schedule from prefect.schedules.clocks import IntervalClock + from pipelines.constants import constants hour_schedule = Schedule( diff --git a/pipelines/meteorologia/meteorologia_inmet/tasks.py b/pipelines/meteorologia/meteorologia_inmet/tasks.py index 98715a63..14a7fcfa 100644 --- a/pipelines/meteorologia/meteorologia_inmet/tasks.py +++ b/pipelines/meteorologia/meteorologia_inmet/tasks.py @@ -2,21 +2,19 @@ """ Tasks for meteorologia_inmet """ -from datetime import datetime, timedelta import json +from datetime import datetime, timedelta from pathlib import Path from typing import Tuple, Union import pandas as pd import pendulum -from prefect import task import requests +from prefect import task from pipelines.constants import constants +from pipelines.meteorologia.precipitacao_alertario.utils import parse_date_columns from pipelines.utils.utils import get_vault_secret, log, to_partitions -from pipelines.meteorologia.precipitacao_alertario.utils import ( - parse_date_columns, -) # from pipelines.rj_cor.meteorologia.meteorologia_inmet.meteorologia_utils import converte_timezone diff --git a/pipelines/meteorologia/meteorologia_redemet/flows.py b/pipelines/meteorologia/meteorologia_redemet/flows.py index b6f8b07e..04e8bc00 100644 --- a/pipelines/meteorologia/meteorologia_redemet/flows.py +++ b/pipelines/meteorologia/meteorologia_redemet/flows.py @@ -5,26 +5,26 @@ """ from datetime import timedelta -from prefect import case, Parameter +from prefect import Parameter, case from prefect.run_configs import KubernetesRun from prefect.storage import GCS from prefect.tasks.prefect import create_flow_run, wait_for_flow_run from pipelines.constants import constants -from pipelines.utils.constants import constants as utils_constants +from pipelines.meteorologia.meteorologia_redemet.schedules import ( + hour_schedule, + month_schedule, +) from pipelines.meteorologia.meteorologia_redemet.tasks import ( check_for_new_stations, - get_dates, download_data, download_stations_data, + get_dates, + save_data, treat_data, treat_stations_data, - save_data, -) -from pipelines.meteorologia.meteorologia_redemet.schedules import ( - hour_schedule, - month_schedule, ) +from pipelines.utils.constants import constants as utils_constants from pipelines.utils.decorators import Flow from pipelines.utils.dump_db.constants import constants as dump_db_constants from pipelines.utils.dump_to_gcs.constants import constants as dump_to_gcs_constants diff --git a/pipelines/meteorologia/meteorologia_redemet/schedules.py b/pipelines/meteorologia/meteorologia_redemet/schedules.py index b11153f3..4ed1f739 100644 --- a/pipelines/meteorologia/meteorologia_redemet/schedules.py +++ b/pipelines/meteorologia/meteorologia_redemet/schedules.py @@ -3,9 +3,11 @@ Schedules for meteorologia_redemet Dados são atualizados a cada 1 hora """ -from datetime import timedelta, datetime +from datetime import datetime, timedelta + from prefect.schedules import Schedule from prefect.schedules.clocks import IntervalClock + from pipelines.constants import constants hour_schedule = Schedule( diff --git a/pipelines/meteorologia/meteorologia_redemet/tasks.py b/pipelines/meteorologia/meteorologia_redemet/tasks.py index b429e35c..2ca918e7 100644 --- a/pipelines/meteorologia/meteorologia_redemet/tasks.py +++ b/pipelines/meteorologia/meteorologia_redemet/tasks.py @@ -2,25 +2,25 @@ """ Tasks for meteorologia_redemet """ -from datetime import timedelta import json +from datetime import timedelta from pathlib import Path from typing import Tuple, Union -from unidecode import unidecode import pandas as pd import pendulum +import requests from prefect import task from prefect.engine.signals import ENDRUN from prefect.engine.state import Failed -import requests +from unidecode import unidecode from pipelines.constants import constants from pipelines.utils.utils import ( get_vault_secret, log, - to_partitions, parse_date_columns, + to_partitions, ) diff --git a/pipelines/meteorologia/precipitacao_alertario/flows.py b/pipelines/meteorologia/precipitacao_alertario/flows.py index 65e519e8..26d5d4cf 100644 --- a/pipelines/meteorologia/precipitacao_alertario/flows.py +++ b/pipelines/meteorologia/precipitacao_alertario/flows.py @@ -5,33 +5,31 @@ """ from datetime import timedelta -from prefect import case, Parameter +from prefect import Parameter, case from prefect.executors import LocalDaskExecutor from prefect.run_configs import KubernetesRun from prefect.storage import GCS from prefect.tasks.prefect import create_flow_run, wait_for_flow_run from pipelines.constants import constants -from pipelines.utils.constants import constants as utils_constants -from pipelines.utils.custom import wait_for_flow_run_with_timeout from pipelines.meteorologia.precipitacao_alertario.constants import ( constants as alertario_constants, ) +from pipelines.meteorologia.precipitacao_alertario.schedules import minute_schedule from pipelines.meteorologia.precipitacao_alertario.tasks import ( check_to_run_dbt, download_data, - treat_old_pluviometer, - treat_pluviometer_and_meteorological_data, save_data, save_data_old, save_last_dbt_update, -) -from pipelines.meteorologia.precipitacao_alertario.schedules import ( - minute_schedule, + treat_old_pluviometer, + treat_pluviometer_and_meteorological_data, ) from pipelines.rj_escritorio.rain_dashboard.constants import ( constants as rain_dashboard_constants, ) +from pipelines.utils.constants import constants as utils_constants +from pipelines.utils.custom import wait_for_flow_run_with_timeout from pipelines.utils.decorators import Flow from pipelines.utils.dump_db.constants import constants as dump_db_constants from pipelines.utils.dump_to_gcs.constants import constants as dump_to_gcs_constants diff --git a/pipelines/meteorologia/precipitacao_alertario/schedules.py b/pipelines/meteorologia/precipitacao_alertario/schedules.py index 44fd46e7..104e534b 100644 --- a/pipelines/meteorologia/precipitacao_alertario/schedules.py +++ b/pipelines/meteorologia/precipitacao_alertario/schedules.py @@ -4,9 +4,11 @@ Schedules for precipitacao_alertario Rodar a cada 1 minuto """ -from datetime import timedelta, datetime +from datetime import datetime, timedelta + from prefect.schedules import Schedule from prefect.schedules.clocks import IntervalClock + from pipelines.constants import constants minute_schedule = Schedule( diff --git a/pipelines/meteorologia/precipitacao_alertario/tasks.py b/pipelines/meteorologia/precipitacao_alertario/tasks.py index ea99bd9c..c3e00b7f 100644 --- a/pipelines/meteorologia/precipitacao_alertario/tasks.py +++ b/pipelines/meteorologia/precipitacao_alertario/tasks.py @@ -5,13 +5,13 @@ """ from datetime import timedelta from pathlib import Path -from typing import Union, Tuple -import requests +from typing import Tuple, Union -from bs4 import BeautifulSoup import numpy as np import pandas as pd import pendulum +import requests +from bs4 import BeautifulSoup from prefect import task from pipelines.constants import constants @@ -25,10 +25,10 @@ get_redis_output, get_vault_secret, log, - to_partitions, parse_date_columns, save_str_on_redis, save_updated_rows_on_redis, + to_partitions, ) diff --git a/pipelines/meteorologia/precipitacao_cemaden/flows.py b/pipelines/meteorologia/precipitacao_cemaden/flows.py index 14c81a6b..cbf4e2e9 100644 --- a/pipelines/meteorologia/precipitacao_cemaden/flows.py +++ b/pipelines/meteorologia/precipitacao_cemaden/flows.py @@ -5,29 +5,27 @@ """ from datetime import timedelta -from prefect import case, Parameter +from prefect import Parameter, case from prefect.run_configs import KubernetesRun from prefect.storage import GCS from prefect.tasks.prefect import create_flow_run, wait_for_flow_run from pipelines.constants import constants -from pipelines.utils.constants import constants as utils_constants -from pipelines.utils.custom import wait_for_flow_run_with_timeout from pipelines.meteorologia.precipitacao_cemaden.constants import ( constants as cemaden_constants, ) +from pipelines.meteorologia.precipitacao_cemaden.schedules import minute_schedule from pipelines.meteorologia.precipitacao_cemaden.tasks import ( check_for_new_stations, download_data, - treat_data, save_data, -) -from pipelines.meteorologia.precipitacao_cemaden.schedules import ( - minute_schedule, + treat_data, ) from pipelines.rj_escritorio.rain_dashboard.constants import ( constants as rain_dashboard_constants, ) +from pipelines.utils.constants import constants as utils_constants +from pipelines.utils.custom import wait_for_flow_run_with_timeout from pipelines.utils.decorators import Flow from pipelines.utils.dump_db.constants import constants as dump_db_constants from pipelines.utils.dump_to_gcs.constants import constants as dump_to_gcs_constants diff --git a/pipelines/meteorologia/precipitacao_cemaden/schedules.py b/pipelines/meteorologia/precipitacao_cemaden/schedules.py index 6b56ffe9..eb24de1e 100644 --- a/pipelines/meteorologia/precipitacao_cemaden/schedules.py +++ b/pipelines/meteorologia/precipitacao_cemaden/schedules.py @@ -4,9 +4,11 @@ Schedules for precipitacao_cemaden Rodar a cada 1 minuto """ -from datetime import timedelta, datetime +from datetime import datetime, timedelta + from prefect.schedules import Schedule from prefect.schedules.clocks import IntervalClock + from pipelines.constants import constants minute_schedule = Schedule( diff --git a/pipelines/meteorologia/precipitacao_cemaden/tasks.py b/pipelines/meteorologia/precipitacao_cemaden/tasks.py index c7c3ee09..a7bf44f1 100644 --- a/pipelines/meteorologia/precipitacao_cemaden/tasks.py +++ b/pipelines/meteorologia/precipitacao_cemaden/tasks.py @@ -5,14 +5,15 @@ """ from datetime import timedelta from pathlib import Path -from typing import Union, Tuple +from typing import Tuple, Union import numpy as np import pandas as pd import pendulum from prefect import task from prefect.engine.signals import ENDRUN -from prefect.engine.state import Skipped, Failed +from prefect.engine.state import Failed, Skipped + from pipelines.constants import constants from pipelines.utils.utils import ( log, diff --git a/pipelines/meteorologia/precipitacao_inea/flows.py b/pipelines/meteorologia/precipitacao_inea/flows.py index 112e8e81..aff64270 100644 --- a/pipelines/meteorologia/precipitacao_inea/flows.py +++ b/pipelines/meteorologia/precipitacao_inea/flows.py @@ -5,25 +5,23 @@ """ from datetime import timedelta -from prefect import case, Parameter +from prefect import Parameter, case from prefect.run_configs import KubernetesRun from prefect.storage import GCS from prefect.tasks.prefect import create_flow_run from pipelines.constants import constants -from pipelines.utils.constants import constants as utils_constants -from pipelines.utils.custom import wait_for_flow_run_with_timeout +from pipelines.meteorologia.precipitacao_inea.schedules import minute_schedule from pipelines.meteorologia.precipitacao_inea.tasks import ( check_for_new_stations, check_new_data, download_data, - treat_data, save_data, + treat_data, wait_task, ) -from pipelines.meteorologia.precipitacao_inea.schedules import ( - minute_schedule, -) +from pipelines.utils.constants import constants as utils_constants +from pipelines.utils.custom import wait_for_flow_run_with_timeout from pipelines.utils.decorators import Flow from pipelines.utils.dump_db.constants import constants as dump_db_constants from pipelines.utils.dump_to_gcs.constants import constants as dump_to_gcs_constants diff --git a/pipelines/meteorologia/precipitacao_inea/schedules.py b/pipelines/meteorologia/precipitacao_inea/schedules.py index f16b41aa..586d9d9f 100644 --- a/pipelines/meteorologia/precipitacao_inea/schedules.py +++ b/pipelines/meteorologia/precipitacao_inea/schedules.py @@ -4,9 +4,11 @@ Schedules for precipitacao_inea Rodar a cada 1 minuto """ -from datetime import timedelta, datetime +from datetime import datetime, timedelta + from prefect.schedules import Schedule from prefect.schedules.clocks import IntervalClock + from pipelines.constants import constants minute_schedule = Schedule( diff --git a/pipelines/meteorologia/precipitacao_inea/tasks.py b/pipelines/meteorologia/precipitacao_inea/tasks.py index 836f84c2..d031a10b 100644 --- a/pipelines/meteorologia/precipitacao_inea/tasks.py +++ b/pipelines/meteorologia/precipitacao_inea/tasks.py @@ -5,14 +5,15 @@ """ from datetime import timedelta from pathlib import Path -from typing import Union, Tuple +from typing import Tuple, Union import numpy as np import pandas as pd import pendulum from prefect import task from prefect.engine.signals import ENDRUN -from prefect.engine.state import Skipped, Failed +from prefect.engine.state import Failed, Skipped + from pipelines.constants import constants from pipelines.utils.utils import ( log, diff --git a/pipelines/meteorologia/precipitacao_websirene/flows.py b/pipelines/meteorologia/precipitacao_websirene/flows.py index 71826af7..f3296521 100644 --- a/pipelines/meteorologia/precipitacao_websirene/flows.py +++ b/pipelines/meteorologia/precipitacao_websirene/flows.py @@ -3,19 +3,17 @@ """ Flows for precipitacao_alertario """ -from prefect import case, Parameter +from prefect import Parameter, case from prefect.run_configs import KubernetesRun from prefect.storage import GCS from prefect.tasks.prefect import create_flow_run, wait_for_flow_run from pipelines.constants import constants +from pipelines.meteorologia.precipitacao_websirene.schedules import MINUTE_SCHEDULE from pipelines.meteorologia.precipitacao_websirene.tasks import ( download_dados, - tratar_dados, salvar_dados, -) -from pipelines.meteorologia.precipitacao_websirene.schedules import ( - MINUTE_SCHEDULE, + tratar_dados, ) from pipelines.utils.constants import constants as utils_constants from pipelines.utils.decorators import Flow @@ -24,7 +22,6 @@ get_current_flow_labels, ) - with Flow( "COR: Meteorologia - Precipitacao WEBSIRENE", code_owners=[ diff --git a/pipelines/meteorologia/precipitacao_websirene/schedules.py b/pipelines/meteorologia/precipitacao_websirene/schedules.py index 0c3babd3..db267f2b 100644 --- a/pipelines/meteorologia/precipitacao_websirene/schedules.py +++ b/pipelines/meteorologia/precipitacao_websirene/schedules.py @@ -3,9 +3,11 @@ Schedules for precipitacao_alertario Rodar a cada 30 ou 15 minutos """ -from datetime import timedelta, datetime +from datetime import datetime, timedelta + from prefect.schedules import Schedule from prefect.schedules.clocks import IntervalClock + from pipelines.constants import constants MINUTE_SCHEDULE = Schedule( diff --git a/pipelines/meteorologia/precipitacao_websirene/tasks.py b/pipelines/meteorologia/precipitacao_websirene/tasks.py index 4737762b..5923427c 100644 --- a/pipelines/meteorologia/precipitacao_websirene/tasks.py +++ b/pipelines/meteorologia/precipitacao_websirene/tasks.py @@ -4,19 +4,19 @@ """ from datetime import timedelta from pathlib import Path -from typing import Union, Tuple +from typing import Tuple, Union import pandas as pd +import pandas_read_xml as pdx import pendulum from prefect import task -import pandas_read_xml as pdx from pipelines.constants import constants from pipelines.utils.utils import ( log, - to_partitions, parse_date_columns, save_updated_rows_on_redis, + to_partitions, ) diff --git a/pipelines/meteorologia/radar/mendanha/flows.py b/pipelines/meteorologia/radar/mendanha/flows.py index b77e5d08..5dcb4288 100644 --- a/pipelines/meteorologia/radar/mendanha/flows.py +++ b/pipelines/meteorologia/radar/mendanha/flows.py @@ -19,28 +19,23 @@ # from pipelines.tasks import task_get_redis_client # from pipelines.meteorologia.radar.mendanha.schedules import TIME_SCHEDULE from pipelines.meteorologia.radar.mendanha.constants import constants as radar_constants -from pipelines.meteorologia.radar.mendanha.tasks import ( # prefix_to_restore,; save_data, +from pipelines.meteorologia.radar.mendanha.tasks import ( # prefix_to_restore,; save_data,; create_visualization_with_background,; get_storage_destination,; upload_file_to_storage,; prefix_to_restore,; save_data, access_api, add_new_image, base64_to_bytes, combine_radar_files, compress_to_zip, create_visualization_no_background, - # create_visualization_with_background, download_files_storage, get_and_format_time, get_colorbar_title, get_filenames_storage, get_radar_parameters, - # get_storage_destination, img_to_base64, remap_data, rename_keys_redis, save_images_to_local, save_img_on_redis, - # upload_file_to_storage, - # prefix_to_restore, - # save_data, ) from pipelines.utils_rj_cor import build_redis_key @@ -53,7 +48,6 @@ # from prefect.tasks.prefect import create_flow_run, wait_for_flow_run - # from pipelines.tasks import ( # get_on_redis, # save_on_redis, diff --git a/pipelines/meteorologia/radar/mendanha/tasks.py b/pipelines/meteorologia/radar/mendanha/tasks.py index 21adfe0d..8cdc7b74 100644 --- a/pipelines/meteorologia/radar/mendanha/tasks.py +++ b/pipelines/meteorologia/radar/mendanha/tasks.py @@ -14,9 +14,6 @@ from typing import Dict, List, Tuple, Union import cartopy.crs as ccrs - -# import contextily as ctx -from google.cloud import storage import matplotlib.pyplot as plt import numpy as np import pendulum @@ -24,6 +21,8 @@ # import wradlib as wrl import xarray as xr + +# import contextily as ctx from google.cloud import storage # from mpl_toolkits.axes_grid1 import make_axes_locatable @@ -52,8 +51,6 @@ # from pyart.map import grid_from_radars - - @task() def get_filenames_storage( bucket_name: str = "rj-escritorio-scp", diff --git a/pipelines/meteorologia/satelite/flows.py b/pipelines/meteorologia/satelite/flows.py index a5d47117..64df7a34 100644 --- a/pipelines/meteorologia/satelite/flows.py +++ b/pipelines/meteorologia/satelite/flows.py @@ -52,9 +52,6 @@ # from prefect.tasks.prefect import create_flow_run, wait_for_flow_run - - - with Flow( name="COR: Meteorologia - Satelite GOES 16", state_handlers=[ diff --git a/pipelines/meteorologia/satelite/satellite_utils.py b/pipelines/meteorologia/satelite/satellite_utils.py index a0a11a21..a3307e0a 100644 --- a/pipelines/meteorologia/satelite/satellite_utils.py +++ b/pipelines/meteorologia/satelite/satellite_utils.py @@ -74,11 +74,6 @@ ) from pipelines.meteorologia.satelite.remap import remap -from refeitura_rio.pipelines_templates.dump_url import ( - get_credentials_from_env, -) # pylint: disable=E0401 -from refeitura_rio.pipelines_utils.bd import list_blobs_with_prefix # pylint: disable=E0401 -from prefeitura_rio.pipelines_utils.logging import log def get_blob_with_prefix(bucket_name: str, prefix: str, mode: str = "prod") -> str: diff --git a/pipelines/meteorologia/satelite/tasks.py b/pipelines/meteorologia/satelite/tasks.py index 3c50f50b..4753e64f 100644 --- a/pipelines/meteorologia/satelite/tasks.py +++ b/pipelines/meteorologia/satelite/tasks.py @@ -37,8 +37,6 @@ # import requests - - @task() def get_dates(current_time, product) -> str: """ diff --git a/pipelines/meteorologia/utils.py b/pipelines/meteorologia/utils.py index 5d6068e9..9b5d7180 100644 --- a/pipelines/meteorologia/utils.py +++ b/pipelines/meteorologia/utils.py @@ -5,10 +5,7 @@ """ import pandas as pd -from pipelines.utils.utils import ( - get_redis_client, - log, -) +from pipelines.utils.utils import get_redis_client, log def save_updated_rows_on_redis( diff --git a/pipelines/tasks.py b/pipelines/tasks.py index 20cc2c2a..509c933d 100644 --- a/pipelines/tasks.py +++ b/pipelines/tasks.py @@ -5,8 +5,8 @@ """ from pathlib import Path -from typing import List, Union, Tuple -from google.cloud import storage +from typing import List, Tuple, Union + import pandas as pd import pendulum from google.cloud import storage @@ -18,7 +18,9 @@ parse_date_columns, to_partitions, ) -from prefeitura_rio.pipelines_utils.redis_pal import get_redis_client # pylint: disable=E0611, E0401 +from prefeitura_rio.pipelines_utils.redis_pal import ( # pylint: disable=E0611, E0401 + get_redis_client, +) from pipelines.utils_rj_cor import build_redis_key diff --git a/pipelines/teste_pipeline/flows.py b/pipelines/teste_pipeline/flows.py index 4304cf13..e6659f70 100644 --- a/pipelines/teste_pipeline/flows.py +++ b/pipelines/teste_pipeline/flows.py @@ -7,11 +7,7 @@ from prefect.run_configs import LocalRun -from pipelines.teste_pipeline.tasks import ( - get_fake_data, - treat_fake_data, - print_data, -) +from pipelines.teste_pipeline.tasks import get_fake_data, print_data, treat_fake_data from pipelines.utils.decorators import Flow with Flow( diff --git a/pipelines/teste_pipeline/schedules.py b/pipelines/teste_pipeline/schedules.py index 053b4a7c..8d27a64e 100644 --- a/pipelines/teste_pipeline/schedules.py +++ b/pipelines/teste_pipeline/schedules.py @@ -3,11 +3,11 @@ Schedules for cor """ -from datetime import timedelta, datetime, time +from datetime import datetime, time, timedelta +import pytz from prefect.schedules import Schedule from prefect.schedules.clocks import IntervalClock -import pytz from pipelines.constants import constants diff --git a/pipelines/teste_pipeline/tasks.py b/pipelines/teste_pipeline/tasks.py index b50bac40..0b5e504d 100644 --- a/pipelines/teste_pipeline/tasks.py +++ b/pipelines/teste_pipeline/tasks.py @@ -3,6 +3,7 @@ Tasks for cor """ from typing import List + from prefect import task diff --git a/pipelines/utils/utils.py b/pipelines/utils/utils.py index 0fe8c8fb..52362909 100644 --- a/pipelines/utils/utils.py +++ b/pipelines/utils/utils.py @@ -25,6 +25,7 @@ import pendulum import prefect +# from redis_pal import RedisPal # from redis_pal import RedisPal import requests from google.cloud import storage @@ -33,12 +34,7 @@ from prefect.client import Client from prefect.engine.state import Skipped, State from prefect.run_configs import KubernetesRun -from prefect.utilities.graphql import ( - with_args, -) - -# from redis_pal import RedisPal -import requests +from prefect.utilities.graphql import with_args # import telegram from prefeitura_rio.pipelines_utils.redis_pal import get_redis_client diff --git a/pipelines/utils_rj_cor.py b/pipelines/utils_rj_cor.py index 0b0d144d..a28fccb7 100644 --- a/pipelines/utils_rj_cor.py +++ b/pipelines/utils_rj_cor.py @@ -13,11 +13,14 @@ # from redis_pal import RedisPal # import pipelines.constants -from prefeitura_rio.pipelines_utils.infisical import get_secret # pylint: disable=E0611, E0401 -from prefeitura_rio.pipelines_utils.redis_pal import get_redis_client # pylint: disable=E0611, E0401 -from pipelines.utils.utils import ( - log, +from prefeitura_rio.pipelines_utils.infisical import ( + get_secret, # pylint: disable=E0611, E0401 ) +from prefeitura_rio.pipelines_utils.redis_pal import ( # pylint: disable=E0611, E0401 + get_redis_client, +) + +from pipelines.utils.utils import log ############### #