Skip to content

Commit

Permalink
[pre-commit.ci] auto fixes from pre-commit.com hooks
Browse files Browse the repository at this point in the history
for more information, see https://pre-commit.ci
  • Loading branch information
pre-commit-ci[bot] committed Sep 13, 2024
1 parent 04f48c9 commit a63b967
Show file tree
Hide file tree
Showing 33 changed files with 96 additions and 123 deletions.
2 changes: 1 addition & 1 deletion .github/ISSUE_TEMPLATE/notifica--o-de-problema.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name: Notificação de problema
about: Notifique um problema que está ocorrendo para que possamos corrigí-lo.
title: "[BUG]"
labels: bug
assignees:
assignees:
---

**Descreva o problema**
Expand Down
2 changes: 1 addition & 1 deletion .github/ISSUE_TEMPLATE/requerimento-de-funcionalidade.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name: Requerimento de funcionalidade
about: Sugira novas funcionalidades que podem ser úteis a todos os desenvolvedores.
title: "[FEATURE]"
labels: enhancement
assignees:
assignees:
---

**Essa funcionalidade está relacionada a um problema?**
Expand Down
6 changes: 1 addition & 5 deletions .github/workflows/scripts/register_flows.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,11 +91,7 @@ def build_and_register( # pylint: disable=too-many-branches
while attempts < max_retries:
attempts += 1
try:
(
flow_id,
flow_version,
is_new,
) = register_serialized_flow(
(flow_id, flow_version, is_new,) = register_serialized_flow(
client=client,
serialized_flow=serialized_flow,
project_id=project_id,
Expand Down
14 changes: 6 additions & 8 deletions pipelines/meteorologia/meteorologia_inmet/flows.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,20 @@
"""
from datetime import timedelta

from prefect import case, Parameter
from prefect import Parameter, case
from prefect.run_configs import KubernetesRun
from prefect.storage import GCS
from prefect.tasks.prefect import create_flow_run, wait_for_flow_run

from pipelines.constants import constants
from pipelines.utils.constants import constants as utils_constants
from pipelines.meteorologia.meteorologia_inmet.tasks import (
get_dates,
# slice_data,
from pipelines.meteorologia.meteorologia_inmet.schedules import hour_schedule
from pipelines.meteorologia.meteorologia_inmet.tasks import ( # slice_data,
download,
tratar_dados,
get_dates,
salvar_dados,
tratar_dados,
)
from pipelines.meteorologia.meteorologia_inmet.schedules import hour_schedule
from pipelines.utils.constants import constants as utils_constants
from pipelines.utils.decorators import Flow
from pipelines.utils.dump_db.constants import constants as dump_db_constants
from pipelines.utils.dump_to_gcs.constants import constants as dump_to_gcs_constants
Expand All @@ -28,7 +27,6 @@
get_current_flow_labels,
)


with Flow(
name="COR: Meteorologia - Meteorologia INMET",
code_owners=[
Expand Down
4 changes: 3 additions & 1 deletion pipelines/meteorologia/meteorologia_inmet/schedules.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@
Schedules for meteorologia_inmet
Rodar a cada 1 hora
"""
from datetime import timedelta, datetime
from datetime import datetime, timedelta

from prefect.schedules import Schedule
from prefect.schedules.clocks import IntervalClock

from pipelines.constants import constants

hour_schedule = Schedule(
Expand Down
8 changes: 3 additions & 5 deletions pipelines/meteorologia/meteorologia_inmet/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,19 @@
"""
Tasks for meteorologia_inmet
"""
from datetime import datetime, timedelta
import json
from datetime import datetime, timedelta
from pathlib import Path
from typing import Tuple, Union

import pandas as pd
import pendulum
from prefect import task
import requests
from prefect import task

from pipelines.constants import constants
from pipelines.meteorologia.precipitacao_alertario.utils import parse_date_columns
from pipelines.utils.utils import get_vault_secret, log, to_partitions
from pipelines.meteorologia.precipitacao_alertario.utils import (
parse_date_columns,
)

# from pipelines.rj_cor.meteorologia.meteorologia_inmet.meteorologia_utils import converte_timezone

Expand Down
16 changes: 8 additions & 8 deletions pipelines/meteorologia/meteorologia_redemet/flows.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,26 +5,26 @@
"""
from datetime import timedelta

from prefect import case, Parameter
from prefect import Parameter, case
from prefect.run_configs import KubernetesRun
from prefect.storage import GCS
from prefect.tasks.prefect import create_flow_run, wait_for_flow_run

from pipelines.constants import constants
from pipelines.utils.constants import constants as utils_constants
from pipelines.meteorologia.meteorologia_redemet.schedules import (
hour_schedule,
month_schedule,
)
from pipelines.meteorologia.meteorologia_redemet.tasks import (
check_for_new_stations,
get_dates,
download_data,
download_stations_data,
get_dates,
save_data,
treat_data,
treat_stations_data,
save_data,
)
from pipelines.meteorologia.meteorologia_redemet.schedules import (
hour_schedule,
month_schedule,
)
from pipelines.utils.constants import constants as utils_constants
from pipelines.utils.decorators import Flow
from pipelines.utils.dump_db.constants import constants as dump_db_constants
from pipelines.utils.dump_to_gcs.constants import constants as dump_to_gcs_constants
Expand Down
4 changes: 3 additions & 1 deletion pipelines/meteorologia/meteorologia_redemet/schedules.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@
Schedules for meteorologia_redemet
Dados são atualizados a cada 1 hora
"""
from datetime import timedelta, datetime
from datetime import datetime, timedelta

from prefect.schedules import Schedule
from prefect.schedules.clocks import IntervalClock

from pipelines.constants import constants

hour_schedule = Schedule(
Expand Down
8 changes: 4 additions & 4 deletions pipelines/meteorologia/meteorologia_redemet/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,25 +2,25 @@
"""
Tasks for meteorologia_redemet
"""
from datetime import timedelta
import json
from datetime import timedelta
from pathlib import Path
from typing import Tuple, Union
from unidecode import unidecode

import pandas as pd
import pendulum
import requests
from prefect import task
from prefect.engine.signals import ENDRUN
from prefect.engine.state import Failed
import requests
from unidecode import unidecode

from pipelines.constants import constants
from pipelines.utils.utils import (
get_vault_secret,
log,
to_partitions,
parse_date_columns,
to_partitions,
)


Expand Down
14 changes: 6 additions & 8 deletions pipelines/meteorologia/precipitacao_alertario/flows.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,33 +5,31 @@
"""
from datetime import timedelta

from prefect import case, Parameter
from prefect import Parameter, case
from prefect.executors import LocalDaskExecutor
from prefect.run_configs import KubernetesRun
from prefect.storage import GCS
from prefect.tasks.prefect import create_flow_run, wait_for_flow_run

from pipelines.constants import constants
from pipelines.utils.constants import constants as utils_constants
from pipelines.utils.custom import wait_for_flow_run_with_timeout
from pipelines.meteorologia.precipitacao_alertario.constants import (
constants as alertario_constants,
)
from pipelines.meteorologia.precipitacao_alertario.schedules import minute_schedule
from pipelines.meteorologia.precipitacao_alertario.tasks import (
check_to_run_dbt,
download_data,
treat_old_pluviometer,
treat_pluviometer_and_meteorological_data,
save_data,
save_data_old,
save_last_dbt_update,
)
from pipelines.meteorologia.precipitacao_alertario.schedules import (
minute_schedule,
treat_old_pluviometer,
treat_pluviometer_and_meteorological_data,
)
from pipelines.rj_escritorio.rain_dashboard.constants import (
constants as rain_dashboard_constants,
)
from pipelines.utils.constants import constants as utils_constants
from pipelines.utils.custom import wait_for_flow_run_with_timeout
from pipelines.utils.decorators import Flow
from pipelines.utils.dump_db.constants import constants as dump_db_constants
from pipelines.utils.dump_to_gcs.constants import constants as dump_to_gcs_constants
Expand Down
4 changes: 3 additions & 1 deletion pipelines/meteorologia/precipitacao_alertario/schedules.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@
Schedules for precipitacao_alertario
Rodar a cada 1 minuto
"""
from datetime import timedelta, datetime
from datetime import datetime, timedelta

from prefect.schedules import Schedule
from prefect.schedules.clocks import IntervalClock

from pipelines.constants import constants

minute_schedule = Schedule(
Expand Down
8 changes: 4 additions & 4 deletions pipelines/meteorologia/precipitacao_alertario/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@
"""
from datetime import timedelta
from pathlib import Path
from typing import Union, Tuple
import requests
from typing import Tuple, Union

from bs4 import BeautifulSoup
import numpy as np
import pandas as pd
import pendulum
import requests
from bs4 import BeautifulSoup
from prefect import task

from pipelines.constants import constants
Expand All @@ -25,10 +25,10 @@
get_redis_output,
get_vault_secret,
log,
to_partitions,
parse_date_columns,
save_str_on_redis,
save_updated_rows_on_redis,
to_partitions,
)


Expand Down
12 changes: 5 additions & 7 deletions pipelines/meteorologia/precipitacao_cemaden/flows.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,29 +5,27 @@
"""
from datetime import timedelta

from prefect import case, Parameter
from prefect import Parameter, case
from prefect.run_configs import KubernetesRun
from prefect.storage import GCS
from prefect.tasks.prefect import create_flow_run, wait_for_flow_run

from pipelines.constants import constants
from pipelines.utils.constants import constants as utils_constants
from pipelines.utils.custom import wait_for_flow_run_with_timeout
from pipelines.meteorologia.precipitacao_cemaden.constants import (
constants as cemaden_constants,
)
from pipelines.meteorologia.precipitacao_cemaden.schedules import minute_schedule
from pipelines.meteorologia.precipitacao_cemaden.tasks import (
check_for_new_stations,
download_data,
treat_data,
save_data,
)
from pipelines.meteorologia.precipitacao_cemaden.schedules import (
minute_schedule,
treat_data,
)
from pipelines.rj_escritorio.rain_dashboard.constants import (
constants as rain_dashboard_constants,
)
from pipelines.utils.constants import constants as utils_constants
from pipelines.utils.custom import wait_for_flow_run_with_timeout
from pipelines.utils.decorators import Flow
from pipelines.utils.dump_db.constants import constants as dump_db_constants
from pipelines.utils.dump_to_gcs.constants import constants as dump_to_gcs_constants
Expand Down
4 changes: 3 additions & 1 deletion pipelines/meteorologia/precipitacao_cemaden/schedules.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@
Schedules for precipitacao_cemaden
Rodar a cada 1 minuto
"""
from datetime import timedelta, datetime
from datetime import datetime, timedelta

from prefect.schedules import Schedule
from prefect.schedules.clocks import IntervalClock

from pipelines.constants import constants

minute_schedule = Schedule(
Expand Down
5 changes: 3 additions & 2 deletions pipelines/meteorologia/precipitacao_cemaden/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,15 @@
"""
from datetime import timedelta
from pathlib import Path
from typing import Union, Tuple
from typing import Tuple, Union

import numpy as np
import pandas as pd
import pendulum
from prefect import task
from prefect.engine.signals import ENDRUN
from prefect.engine.state import Skipped, Failed
from prefect.engine.state import Failed, Skipped

from pipelines.constants import constants
from pipelines.utils.utils import (
log,
Expand Down
12 changes: 5 additions & 7 deletions pipelines/meteorologia/precipitacao_inea/flows.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,25 +5,23 @@
"""
from datetime import timedelta

from prefect import case, Parameter
from prefect import Parameter, case
from prefect.run_configs import KubernetesRun
from prefect.storage import GCS
from prefect.tasks.prefect import create_flow_run

from pipelines.constants import constants
from pipelines.utils.constants import constants as utils_constants
from pipelines.utils.custom import wait_for_flow_run_with_timeout
from pipelines.meteorologia.precipitacao_inea.schedules import minute_schedule
from pipelines.meteorologia.precipitacao_inea.tasks import (
check_for_new_stations,
check_new_data,
download_data,
treat_data,
save_data,
treat_data,
wait_task,
)
from pipelines.meteorologia.precipitacao_inea.schedules import (
minute_schedule,
)
from pipelines.utils.constants import constants as utils_constants
from pipelines.utils.custom import wait_for_flow_run_with_timeout
from pipelines.utils.decorators import Flow
from pipelines.utils.dump_db.constants import constants as dump_db_constants
from pipelines.utils.dump_to_gcs.constants import constants as dump_to_gcs_constants
Expand Down
4 changes: 3 additions & 1 deletion pipelines/meteorologia/precipitacao_inea/schedules.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@
Schedules for precipitacao_inea
Rodar a cada 1 minuto
"""
from datetime import timedelta, datetime
from datetime import datetime, timedelta

from prefect.schedules import Schedule
from prefect.schedules.clocks import IntervalClock

from pipelines.constants import constants

minute_schedule = Schedule(
Expand Down
5 changes: 3 additions & 2 deletions pipelines/meteorologia/precipitacao_inea/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,15 @@
"""
from datetime import timedelta
from pathlib import Path
from typing import Union, Tuple
from typing import Tuple, Union

import numpy as np
import pandas as pd
import pendulum
from prefect import task
from prefect.engine.signals import ENDRUN
from prefect.engine.state import Skipped, Failed
from prefect.engine.state import Failed, Skipped

from pipelines.constants import constants
from pipelines.utils.utils import (
log,
Expand Down
Loading

0 comments on commit a63b967

Please sign in to comment.