From d6a5c4b8398b8a018b519622333b27ca1fa55fe1 Mon Sep 17 00:00:00 2001 From: David Luo Date: Wed, 16 Oct 2024 14:19:48 -0700 Subject: [PATCH] upgrade fbpcs to per-target typechecking (#2414) Summary: We're migrating fbcode to use Pyre's new architecture. To type check your project: ``` arc pyre check //fbpcs/... ``` You can enable this by adding `typing = True` to individual buck targets OR ``` # in a PACKAGE file load("fbcode_macros//build_defs:python.bzl", "python") python.set_typing(True) ``` will type check all buck targets nested underneath. This diff removes the configuration and replaces it with our new buck based architecture and silences any resulting errors. The test failure is preexisting. Reviewed By: joe1234wu Differential Revision: D64347224 --- fbpcs/bolt/bolt_runner.py | 5 ++- fbpcs/bolt/read_config.py | 5 +++ .../common/feature/pcs_feature_gate_utils.py | 37 +++++++++++++++++++ .../data_processing/pid_preparer/preparer.py | 8 ++++ .../pid_preparer/union_pid_preparer_cpp.py | 20 ++++++++++ .../service/id_spine_combiner.py | 12 ++++++ .../pid_run_protocol_binary_service.py | 14 +++++++ .../pc_instance_ca_certificate_provider.py | 5 +++ .../pc_instance_server_certificate.py | 5 +++ .../tests/pc_pre_validation_cli_test.py | 2 + fbpcs/pl_coordinator/token_validator.py | 3 ++ .../post_processing_handler.py | 4 ++ .../tests/dummy_handler.py | 4 ++ .../entity/pid_mr_config.py | 2 + .../private_computation/service/constants.py | 9 +++++ fbpcs/private_computation/service/mpc/mpc.py | 16 ++++++++ .../service/mpc/mpc_game.py | 1 + .../service/pre_validate_service.py | 37 +++++++++++++++++++ .../service/run_binary_base_service.py | 31 +++++++++++++++- fbpcs/service/workflow_sfn_fb.py | 4 ++ 20 files changed, 221 insertions(+), 3 deletions(-) diff --git a/fbpcs/bolt/bolt_runner.py b/fbpcs/bolt/bolt_runner.py index 7b8e20c48..08f93e1d0 100644 --- a/fbpcs/bolt/bolt_runner.py +++ b/fbpcs/bolt/bolt_runner.py @@ -341,7 +341,10 @@ async def _run_hooks( {stage, None}, {role, None}, {when, None}, {event, None} ) for hook in job.hooks.get( - BoltHookKey(event=e, when=w, stage=s.name if s else s, role=r), [] + # pyre-fixme[6]: For 3rd argument expected `Optional[str]` but got + # `Union[None, PrivateComputationBaseStageFlow, str]`. + BoltHookKey(event=e, when=w, stage=s.name if s else s, role=r), + [], ) ) diff --git a/fbpcs/bolt/read_config.py b/fbpcs/bolt/read_config.py index 4fc82f93e..8ef271666 100644 --- a/fbpcs/bolt/read_config.py +++ b/fbpcs/bolt/read_config.py @@ -14,6 +14,9 @@ from fbpcs.bolt.bolt_runner import BoltRunner from fbpcs.bolt.constants import DEFAULT_POLL_INTERVAL_SEC from fbpcs.bolt.oss_bolt_pcs import BoltPCSClient, BoltPCSCreateInstanceArgs + +# pyre-fixme[21]: Could not find module +# `fbpcs.private_computation_cli.private_computation_service_wrapper`. from fbpcs.private_computation_cli.private_computation_service_wrapper import ( build_private_computation_service, ) @@ -45,6 +48,7 @@ def create_bolt_runner( runner_config["partner_client_config"] ) publisher_client = BoltPCSClient( + # pyre-fixme[16]: Module `fbpcs` has no attribute `private_computation_cli`. build_private_computation_service( publisher_client_config["private_computation"], publisher_client_config["mpc"], @@ -54,6 +58,7 @@ def create_bolt_runner( ) ) partner_client = BoltPCSClient( + # pyre-fixme[16]: Module `fbpcs` has no attribute `private_computation_cli`. build_private_computation_service( partner_client_config["private_computation"], partner_client_config["mpc"], diff --git a/fbpcs/common/feature/pcs_feature_gate_utils.py b/fbpcs/common/feature/pcs_feature_gate_utils.py index 54ca4add3..d0f5d1daa 100644 --- a/fbpcs/common/feature/pcs_feature_gate_utils.py +++ b/fbpcs/common/feature/pcs_feature_gate_utils.py @@ -7,23 +7,43 @@ # pyre-strict from typing import Optional, Set, Type +# pyre-fixme[21]: Could not find module `fbpcs.private_computation.entity.infra_config`. from fbpcs.private_computation.entity.infra_config import PrivateComputationGameType + +# pyre-fixme[21]: Could not find module `fbpcs.private_computation.entity.pcs_feature`. from fbpcs.private_computation.entity.pcs_feature import PCSFeature + +# pyre-fixme[21]: Could not find module +# `fbpcs.private_computation.stage_flows.private_computation_base_stage_flow`. from fbpcs.private_computation.stage_flows.private_computation_base_stage_flow import ( PrivateComputationBaseStageFlow, ) + +# pyre-fixme[21]: Could not find module `fbpcs.private_computation.stage_flows.privat... from fbpcs.private_computation.stage_flows.private_computation_mr_pid_pcf2_lift_stage_flow import ( PrivateComputationMrPidPCF2LiftStageFlow, ) + +# pyre-fixme[21]: Could not find module +# `fbpcs.private_computation.stage_flows.private_computation_mr_stage_flow`. from fbpcs.private_computation.stage_flows.private_computation_mr_stage_flow import ( PrivateComputationMRStageFlow, ) + +# pyre-fixme[21]: Could not find module +# `fbpcs.private_computation.stage_flows.private_computation_pcf2_lift_udp_stage_flow`. from fbpcs.private_computation.stage_flows.private_computation_pcf2_lift_udp_stage_flow import ( PrivateComputationPCF2LiftUDPStageFlow, ) + +# pyre-fixme[21]: Could not find module +# `fbpcs.private_computation.stage_flows.private_computation_pcf2_stage_flow`. from fbpcs.private_computation.stage_flows.private_computation_pcf2_stage_flow import ( PrivateComputationPCF2StageFlow, ) + +# pyre-fixme[21]: Could not find module +# `fbpcs.private_computation.stage_flows.private_computation_stage_flow`. from fbpcs.private_computation.stage_flows.private_computation_stage_flow import ( PrivateComputationStageFlow, ) @@ -31,15 +51,24 @@ def get_stage_flow( + # pyre-fixme[11]: Annotation `PrivateComputationGameType` is not defined as a type. game_type: PrivateComputationGameType, + # pyre-fixme[11]: Annotation `PCSFeature` is not defined as a type. pcs_feature_enums: Set[PCSFeature], + # pyre-fixme[11]: Annotation `PrivateComputationBaseStageFlow` is not defined as + # a type. stage_flow_cls: Optional[Type[PrivateComputationBaseStageFlow]] = None, ) -> Type[PrivateComputationBaseStageFlow]: selected_stage_flow_cls = unwrap_or_default( optional=stage_flow_cls, default=( + # pyre-fixme[16]: Module `private_computation` has no attribute + # `stage_flows`. PrivateComputationPCF2StageFlow + # pyre-fixme[16]: Module `entity` has no attribute `infra_config`. if game_type is PrivateComputationGameType.ATTRIBUTION + # pyre-fixme[16]: Module `private_computation` has no attribute + # `stage_flows`. else PrivateComputationStageFlow ), ) @@ -47,13 +76,21 @@ def get_stage_flow( # warning, enabled feature gating will override stage flow, Please contact PSI team to have a similar adoption if PCSFeature.PRIVATE_ATTRIBUTION_MR_PID in pcs_feature_enums: selected_stage_flow_cls = ( + # pyre-fixme[16]: Module `private_computation` has no attribute + # `stage_flows`. PrivateComputationMRStageFlow + # pyre-fixme[16]: Module `entity` has no attribute `infra_config`. if game_type is PrivateComputationGameType.ATTRIBUTION + # pyre-fixme[16]: Module `private_computation` has no attribute + # `stage_flows`. else PrivateComputationMrPidPCF2LiftStageFlow ) if PCSFeature.PRIVATE_LIFT_UNIFIED_DATA_PROCESS in pcs_feature_enums: selected_stage_flow_cls = ( + # pyre-fixme[16]: Module `private_computation` has no attribute + # `stage_flows`. PrivateComputationPCF2LiftUDPStageFlow + # pyre-fixme[16]: Module `entity` has no attribute `infra_config`. if game_type is PrivateComputationGameType.LIFT else selected_stage_flow_cls ) diff --git a/fbpcs/data_processing/pid_preparer/preparer.py b/fbpcs/data_processing/pid_preparer/preparer.py index b33b8cf66..abd9b7674 100644 --- a/fbpcs/data_processing/pid_preparer/preparer.py +++ b/fbpcs/data_processing/pid_preparer/preparer.py @@ -11,8 +11,13 @@ import pathlib from typing import Optional +# pyre-fixme[21]: Could not find module `fbpcp.entity.container_instance`. from fbpcp.entity.container_instance import ContainerInstance + +# pyre-fixme[21]: Could not find module `fbpcp.service.onedocker`. from fbpcp.service.onedocker import OneDockerService + +# pyre-fixme[21]: Could not find module `fbpcp.service.storage`. from fbpcp.service.storage import StorageService @@ -24,6 +29,7 @@ def prepare( output_path: str, log_path: Optional[pathlib.Path] = None, log_level: int = logging.INFO, + # pyre-fixme[11]: Annotation `StorageService` is not defined as a type. storage_svc: Optional[StorageService] = None, ) -> None: pass @@ -34,10 +40,12 @@ def prepare_on_container( input_path: str, output_path: str, # TODO: Support custom log path + # pyre-fixme[11]: Annotation `OneDockerService` is not defined as a type. onedocker_svc: OneDockerService, binary_version: str, tmp_directory: str = "/tmp/", wait_for_container: bool = True, + # pyre-fixme[11]: Annotation `ContainerInstance` is not defined as a type. ) -> ContainerInstance: pass diff --git a/fbpcs/data_processing/pid_preparer/union_pid_preparer_cpp.py b/fbpcs/data_processing/pid_preparer/union_pid_preparer_cpp.py index 0e14b5748..5b2b037a8 100644 --- a/fbpcs/data_processing/pid_preparer/union_pid_preparer_cpp.py +++ b/fbpcs/data_processing/pid_preparer/union_pid_preparer_cpp.py @@ -15,14 +15,27 @@ import tempfile from typing import Dict, Optional +# pyre-fixme[21]: Could not find module `fbpcp.entity.container_instance`. from fbpcp.entity.container_instance import ContainerInstance, ContainerInstanceStatus + +# pyre-fixme[21]: Could not find module `fbpcp.error.pcp`. from fbpcp.error.pcp import ThrottlingError + +# pyre-fixme[21]: Could not find module `fbpcp.service.onedocker`. from fbpcp.service.onedocker import OneDockerService + +# pyre-fixme[21]: Could not find module `fbpcp.service.storage`. from fbpcp.service.storage import PathType, StorageService +# pyre-fixme[21]: Could not find module `fbpcs.common.service.retry_handler`. from fbpcs.common.service.retry_handler import RetryHandler from fbpcs.data_processing.pid_preparer.preparer import UnionPIDDataPreparerService + +# pyre-fixme[21]: Could not find module `fbpcs.onedocker_binary_names`. from fbpcs.onedocker_binary_names import OneDockerBinaryNames + +# pyre-fixme[21]: Could not find module +# `fbpcs.private_computation.service.run_binary_base_service`. from fbpcs.private_computation.service.run_binary_base_service import ( RunBinaryBaseService, ) @@ -43,6 +56,7 @@ def prepare( output_path: str, log_path: Optional[pathlib.Path] = None, log_level: int = logging.INFO, + # pyre-fixme[11]: Annotation `StorageService` is not defined as a type. storage_svc: Optional[StorageService] = None, ) -> None: if log_path is not None: @@ -95,6 +109,7 @@ def prepare_on_container( self, input_path: str, output_path: str, + # pyre-fixme[11]: Annotation `OneDockerService` is not defined as a type. onedocker_svc: OneDockerService, binary_version: str, max_column_count: int = 1, @@ -102,6 +117,7 @@ def prepare_on_container( container_timeout: Optional[int] = None, wait_for_container: bool = True, env_vars: Optional[Dict[str, str]] = None, + # pyre-fixme[11]: Annotation `ContainerInstance` is not defined as a type. ) -> ContainerInstance: return asyncio.run( self.prepare_on_container_async( @@ -145,6 +161,7 @@ async def prepare_on_container_async( current_retry = 0 status = ContainerInstanceStatus.UNKNOWN + # pyre-fixme[16]: Module `fbpcs` has no attribute `onedocker_binary_names`. exe = OneDockerBinaryNames.UNION_PID_PREPARER.value container = None while status is not ContainerInstanceStatus.COMPLETED: @@ -162,6 +179,7 @@ async def prepare_on_container_async( env_vars=env_vars, ) + # pyre-fixme[16]: Module `fbpcs` has no attribute `common`. with RetryHandler( ThrottlingError, logger=logger, backoff_seconds=30 ) as retry_handler: @@ -175,6 +193,8 @@ async def prepare_on_container_async( # Busy wait until the container is finished if wait_for_container: container = ( + # pyre-fixme[16]: Module `fbpcs` has no attribute + # `private_computation`. await RunBinaryBaseService.wait_for_containers_async( onedocker_svc, [container] ) diff --git a/fbpcs/data_processing/service/id_spine_combiner.py b/fbpcs/data_processing/service/id_spine_combiner.py index 07fc1f1ab..daebb5c48 100644 --- a/fbpcs/data_processing/service/id_spine_combiner.py +++ b/fbpcs/data_processing/service/id_spine_combiner.py @@ -8,10 +8,17 @@ from typing import List, Optional +# pyre-fixme[21]: Could not find module `fbpcp.util.arg_builder`. from fbpcp.util.arg_builder import build_cmd_args +# pyre-fixme[21]: Could not find module `fbpcs.private_computation.service.constants`. from fbpcs.private_computation.service.constants import DEFAULT_SORT_STRATEGY + +# pyre-fixme[21]: Could not find module `fbpcs.private_computation.service.pid_utils`. from fbpcs.private_computation.service.pid_utils import get_sharded_filepath + +# pyre-fixme[21]: Could not find module +# `fbpcs.private_computation.service.run_binary_base_service`. from fbpcs.private_computation.service.run_binary_base_service import ( RunBinaryBaseService, ) @@ -21,6 +28,7 @@ DEFAULT_CONTAINER_TIMEOUT_IN_SEC = 10800 +# pyre-fixme[11]: Annotation `RunBinaryBaseService` is not defined as a type. class IdSpineCombinerService(RunBinaryBaseService): @staticmethod def build_args( @@ -31,6 +39,7 @@ def build_args( tmp_directory: str, protocol_type: str, max_id_column_cnt: int = 1, + # pyre-fixme[16]: Module `fbpcs` has no attribute `private_computation`. sort_strategy: str = DEFAULT_SORT_STRATEGY, # TODO T106159008: padding_size and run_name are only temporarily optional # because Lift does not use them. It should and will be required to use them. @@ -46,8 +55,11 @@ def build_args( # own ThreadPoolExecutor here and instead use async primitives cmd_args_list = [] for shard in range(num_shards): + # pyre-fixme[16]: Module `fbpcs` has no attribute `private_computation`. next_spine_path = get_sharded_filepath(spine_path, shard) + # pyre-fixme[16]: Module `fbpcs` has no attribute `private_computation`. next_data_path = get_sharded_filepath(data_path, shard) + # pyre-fixme[16]: Module `fbpcs` has no attribute `private_computation`. next_output_path = get_sharded_filepath(output_path, shard) cmd_args = build_cmd_args( spine_path=next_spine_path, diff --git a/fbpcs/data_processing/service/pid_run_protocol_binary_service.py b/fbpcs/data_processing/service/pid_run_protocol_binary_service.py index 942c98ca2..a9898e8d8 100644 --- a/fbpcs/data_processing/service/pid_run_protocol_binary_service.py +++ b/fbpcs/data_processing/service/pid_run_protocol_binary_service.py @@ -12,6 +12,9 @@ from fbpcs.onedocker_binary_names import OneDockerBinaryNames from fbpcs.pid.entity.pid_instance import PIDProtocol + +# pyre-fixme[21]: Could not find module +# `fbpcs.private_computation.entity.private_computation_instance`. from fbpcs.private_computation.entity.private_computation_instance import ( PrivateComputationRole, ) @@ -35,6 +38,7 @@ def build_args( output_path: str, port: int, tls_args: TlsArgs, + # pyre-fixme[11]: Annotation `PrivateComputationRole` is not defined as a type. pc_role: PrivateComputationRole, use_row_numbers: bool = False, server_endpoint: Optional[str] = None, @@ -66,12 +70,18 @@ def build_args( if use_row_numbers: cmd_ls.append("--use-row-numbers") + # pyre-fixme[16]: Module `entity` has no attribute + # `private_computation_instance`. if tls_args.ca_cert_path and pc_role is PrivateComputationRole.PARTNER: cmd_ls.append(f"--tls-ca {tls_args.ca_cert_path}") + # pyre-fixme[16]: Module `entity` has no attribute + # `private_computation_instance`. if tls_args.server_cert_path and pc_role is PrivateComputationRole.PUBLISHER: cmd_ls.append(f"--tls-cert {tls_args.server_cert_path}") + # pyre-fixme[16]: Module `entity` has no attribute + # `private_computation_instance`. if tls_args.private_key_path and pc_role is PrivateComputationRole.PUBLISHER: cmd_ls.append(f"--tls-key {tls_args.private_key_path}") @@ -82,10 +92,14 @@ def build_args( @staticmethod def get_binary_name(protocol: PIDProtocol, pc_role: PrivateComputationRole) -> str: + # pyre-fixme[16]: Module `entity` has no attribute + # `private_computation_instance`. if pc_role is PrivateComputationRole.PUBLISHER: binary_name = OneDockerBinaryNames.PID_SERVER.value if protocol is PIDProtocol.UNION_PID_MULTIKEY: binary_name = OneDockerBinaryNames.PID_MULTI_KEY_SERVER.value + # pyre-fixme[16]: Module `entity` has no attribute + # `private_computation_instance`. elif pc_role is PrivateComputationRole.PARTNER: binary_name = OneDockerBinaryNames.PID_CLIENT.value if protocol is PIDProtocol.UNION_PID_MULTIKEY: diff --git a/fbpcs/infra/certificate/pc_instance_ca_certificate_provider.py b/fbpcs/infra/certificate/pc_instance_ca_certificate_provider.py index cb16f35f8..11afe91ba 100644 --- a/fbpcs/infra/certificate/pc_instance_ca_certificate_provider.py +++ b/fbpcs/infra/certificate/pc_instance_ca_certificate_provider.py @@ -9,6 +9,9 @@ from typing import Optional from fbpcs.infra.certificate.certificate_provider import CertificateProvider + +# pyre-fixme[21]: Could not find module +# `fbpcs.private_computation.entity.private_computation_instance`. from fbpcs.private_computation.entity.private_computation_instance import ( PrivateComputationInstance, ) @@ -20,7 +23,9 @@ class PCInstanceCaCertificateProvider(CertificateProvider): from PC instance repo. """ + # pyre-fixme[11]: Annotation `PrivateComputationInstance` is not defined as a type. def __init__(self, pc_instance: PrivateComputationInstance) -> None: + # pyre-fixme[4]: Attribute must be annotated. self.pc_instance = pc_instance def get_certificate(self) -> Optional[str]: diff --git a/fbpcs/infra/certificate/pc_instance_server_certificate.py b/fbpcs/infra/certificate/pc_instance_server_certificate.py index f525a968a..400a9efc6 100644 --- a/fbpcs/infra/certificate/pc_instance_server_certificate.py +++ b/fbpcs/infra/certificate/pc_instance_server_certificate.py @@ -9,6 +9,9 @@ from typing import Optional from fbpcs.infra.certificate.certificate_provider import CertificateProvider + +# pyre-fixme[21]: Could not find module +# `fbpcs.private_computation.entity.private_computation_instance`. from fbpcs.private_computation.entity.private_computation_instance import ( PrivateComputationInstance, ) @@ -20,7 +23,9 @@ class PCInstanceServerCertificateProvider(CertificateProvider): from PC instance repo. """ + # pyre-fixme[11]: Annotation `PrivateComputationInstance` is not defined as a type. def __init__(self, pc_instance: PrivateComputationInstance) -> None: + # pyre-fixme[4]: Attribute must be annotated. self.pc_instance = pc_instance def get_certificate(self) -> Optional[str]: diff --git a/fbpcs/pc_pre_validation/tests/pc_pre_validation_cli_test.py b/fbpcs/pc_pre_validation/tests/pc_pre_validation_cli_test.py index ca4410568..e6bae8b14 100644 --- a/fbpcs/pc_pre_validation/tests/pc_pre_validation_cli_test.py +++ b/fbpcs/pc_pre_validation/tests/pc_pre_validation_cli_test.py @@ -91,6 +91,8 @@ def test_parsing_all_args( expected_access_key_id = "access_key_id2" expected_access_key_data = "access_key_data3" expected_binary_version = "binary_version" + # pyre-fixme[9]: expected_pc_computation_role has type + # `PrivateComputationRole`; used as `str`. expected_pc_computation_role: PrivateComputationRole = ( PrivateComputationRole.PARTNER.name ) diff --git a/fbpcs/pl_coordinator/token_validator.py b/fbpcs/pl_coordinator/token_validator.py index 0f3a20232..9673959e7 100644 --- a/fbpcs/pl_coordinator/token_validator.py +++ b/fbpcs/pl_coordinator/token_validator.py @@ -77,6 +77,9 @@ def validate_rule(self, rule: TokenValidationRule) -> None: instance_id="NO_INSTANCE", checkpoint_name="FAIL_FAST_VALIDATION", status=CheckpointStatus.FAILED, + # pyre-fixme[6]: For 5th argument expected + # `Optional[Dict[str, str]]` but got `Dict[str, + # Union[TokenValidationRuleType, str]]`. checkpoint_data={ "rule_name": rule.name, "rule_type": rule.rule_type, diff --git a/fbpcs/post_processing_handler/post_processing_handler.py b/fbpcs/post_processing_handler/post_processing_handler.py index c3ccf9944..e7723657c 100644 --- a/fbpcs/post_processing_handler/post_processing_handler.py +++ b/fbpcs/post_processing_handler/post_processing_handler.py @@ -11,6 +11,8 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: + # pyre-fixme[21]: Could not find module + # `fbpcs.private_computation.entity.private_computation_instance`. from fbpcs.private_computation.entity.private_computation_instance import ( PrivateComputationInstance, ) @@ -29,6 +31,8 @@ class PostProcessingHandler(abc.ABC): async def run( self, storage_svc: StorageService, + # pyre-fixme[11]: Annotation `PrivateComputationInstance` is not defined as + # a type. private_computation_instance: "PrivateComputationInstance", ) -> None: raise NotImplementedError diff --git a/fbpcs/post_processing_handler/tests/dummy_handler.py b/fbpcs/post_processing_handler/tests/dummy_handler.py index de0a24bb3..4c9538bd3 100644 --- a/fbpcs/post_processing_handler/tests/dummy_handler.py +++ b/fbpcs/post_processing_handler/tests/dummy_handler.py @@ -15,6 +15,8 @@ from fbpcs.post_processing_handler.post_processing_handler import PostProcessingHandler if TYPE_CHECKING: + # pyre-fixme[21]: Could not find module + # `fbpcs.private_computation.entity.private_computation_instance`. from fbpcs.private_computation.entity.private_computation_instance import ( PrivateComputationInstance, ) @@ -38,6 +40,8 @@ def __init__( async def run( self, storage_svc: StorageService, + # pyre-fixme[11]: Annotation `PrivateComputationInstance` is not defined as + # a type. private_computation_instance: "PrivateComputationInstance", ) -> None: if random.random() >= self.probability_of_failure: diff --git a/fbpcs/private_computation/entity/pid_mr_config.py b/fbpcs/private_computation/entity/pid_mr_config.py index 846597f6a..6b1f56613 100644 --- a/fbpcs/private_computation/entity/pid_mr_config.py +++ b/fbpcs/private_computation/entity/pid_mr_config.py @@ -10,6 +10,7 @@ from dataclasses import dataclass from enum import Enum +# pyre-fixme[21]: Could not find module `dataclasses_json`. from dataclasses_json import dataclass_json, DataClassJsonMixin @@ -26,6 +27,7 @@ class PidWorkflowConfigs: @dataclass +# pyre-fixme[11]: Annotation `DataClassJsonMixin` is not defined as a type. class PidMrConfig(DataClassJsonMixin): runConfigs: PidRunConfigs workflowConfigs: PidWorkflowConfigs diff --git a/fbpcs/private_computation/service/constants.py b/fbpcs/private_computation/service/constants.py index 28ef811f7..0f3a9162f 100644 --- a/fbpcs/private_computation/service/constants.py +++ b/fbpcs/private_computation/service/constants.py @@ -8,8 +8,10 @@ import typing +# pyre-fixme[21]: Could not find module `fbpcs.pid.entity.pid_instance`. from fbpcs.pid.entity.pid_instance import PIDProtocol +# pyre-fixme[21]: Could not find module `fbpcs.private_computation.entity.infra_config`. from fbpcs.private_computation.entity.infra_config import PrivateComputationGameType """ @@ -32,14 +34,21 @@ DEFAULT_K_ANONYMITY_THRESHOLD_PL = 100 DEFAULT_K_ANONYMITY_THRESHOLD_PA = 0 +# pyre-fixme[11]: Annotation `PIDProtocol` is not defined as a type. +# pyre-fixme[16]: Module `fbpcs` has no attribute `pid`. DEFAULT_PID_PROTOCOL: PIDProtocol = PIDProtocol.UNION_PID DEFAULT_HMAC_KEY: str = "" DEFAULT_CONCURRENCY = 4 ATTRIBUTION_TEST_CONCURRENCY = 1 +# pyre-fixme[11]: Annotation `PrivateComputationGameType` is not defined as a type. DEFAULT_PADDING_SIZE: typing.Dict[PrivateComputationGameType, typing.Optional[int]] = { + # pyre-fixme[16]: Module `entity` has no attribute `infra_config`. PrivateComputationGameType.LIFT: 25, + # pyre-fixme[16]: Module `entity` has no attribute `infra_config`. PrivateComputationGameType.ATTRIBUTION: 4, + # pyre-fixme[16]: Module `entity` has no attribute `infra_config`. PrivateComputationGameType.PRIVATE_ID_DFCA: None, + # pyre-fixme[16]: Module `entity` has no attribute `infra_config`. PrivateComputationGameType.ANONYMIZER: None, } DEFAULT_LOG_COST_TO_S3 = True diff --git a/fbpcs/private_computation/service/mpc/mpc.py b/fbpcs/private_computation/service/mpc/mpc.py index 16e70bffc..f7ce09db6 100644 --- a/fbpcs/private_computation/service/mpc/mpc.py +++ b/fbpcs/private_computation/service/mpc/mpc.py @@ -9,13 +9,22 @@ import logging from typing import Any, Dict, List, Optional, Tuple +# pyre-fixme[21]: Could not find module `fbpcp.service.container`. from fbpcp.service.container import ContainerService + +# pyre-fixme[21]: Could not find module `fbpcp.service.onedocker`. from fbpcp.service.onedocker import OneDockerService + +# pyre-fixme[21]: Could not find module +# `fbpcs.private_computation.entity.private_computation_instance`. from fbpcs.private_computation.entity.private_computation_instance import ( PrivateComputationRole, ) from fbpcs.private_computation.service.mpc.entity.mpc_instance import MPCParty from fbpcs.private_computation.service.mpc.mpc_game import MPCGameService + +# pyre-fixme[21]: Could not find module +# `fbpcs.private_computation.service.run_binary_base_service`. from fbpcs.private_computation.service.run_binary_base_service import ( RunBinaryBaseService, ) @@ -23,6 +32,7 @@ DEFAULT_BINARY_VERSION = "latest" +# pyre-fixme[11]: Annotation `RunBinaryBaseService` is not defined as a type. class MPCService(RunBinaryBaseService): """MPCService is responsible for distributing a larger MPC game to multiple MPC workers @@ -30,6 +40,7 @@ class MPCService(RunBinaryBaseService): def __init__( self, + # pyre-fixme[11]: Annotation `ContainerService` is not defined as a type. container_svc: ContainerService, task_definition: str, mpc_game_svc: MPCGameService, @@ -45,11 +56,13 @@ def __init__( f"Dependency is missing. container_svc={container_svc}, mpc_game_svc={mpc_game_svc}" ) + # pyre-fixme[4]: Attribute must be annotated. self.container_svc = container_svc self.task_definition = task_definition self.mpc_game_svc: MPCGameService = mpc_game_svc self.logger: logging.Logger = logging.getLogger(__name__) + # pyre-fixme[4]: Attribute must be annotated. self.onedocker_svc = OneDockerService(self.container_svc, self.task_definition) """ @@ -138,6 +151,7 @@ def convert_cmd_args_list( def map_private_computation_role_to_mpc_party( + # pyre-fixme[11]: Annotation `PrivateComputationRole` is not defined as a type. private_computation_role: PrivateComputationRole, ) -> MPCParty: """Convert PrivateComputationRole to MPCParty @@ -151,8 +165,10 @@ def map_private_computation_role_to_mpc_party( Exceptions: ValueError: raised when there is no MPCParty associated with private_computation_role """ + # pyre-fixme[16]: Module `private_computation` has no attribute `entity`. if private_computation_role is PrivateComputationRole.PUBLISHER: return MPCParty.SERVER + # pyre-fixme[16]: Module `private_computation` has no attribute `entity`. elif private_computation_role is PrivateComputationRole.PARTNER: return MPCParty.CLIENT else: diff --git a/fbpcs/private_computation/service/mpc/mpc_game.py b/fbpcs/private_computation/service/mpc/mpc_game.py index 1f8145b3c..d0ed4883c 100644 --- a/fbpcs/private_computation/service/mpc/mpc_game.py +++ b/fbpcs/private_computation/service/mpc/mpc_game.py @@ -9,6 +9,7 @@ import logging from typing import Any, Dict, Optional, Tuple +# pyre-fixme[21]: Could not find module `fbpcp.util.arg_builder`. from fbpcp.util.arg_builder import build_cmd_args from fbpcs.private_computation.service.mpc.entity.mpc_game_config import MPCGameConfig diff --git a/fbpcs/private_computation/service/pre_validate_service.py b/fbpcs/private_computation/service/pre_validate_service.py index e13231674..d1dff0995 100644 --- a/fbpcs/private_computation/service/pre_validate_service.py +++ b/fbpcs/private_computation/service/pre_validate_service.py @@ -10,23 +10,48 @@ import logging from typing import Any, Dict, List +# pyre-fixme[21]: Could not find module `fbpcp.entity.container_instance`. from fbpcp.entity.container_instance import ContainerInstanceStatus + +# pyre-fixme[21]: Could not find module `fbpcp.entity.container_type`. from fbpcp.entity.container_type import ContainerType + +# pyre-fixme[21]: Could not find module `fbpcs.onedocker_binary_names`. from fbpcs.onedocker_binary_names import OneDockerBinaryNames + +# pyre-fixme[21]: Could not find module +# `fbpcs.private_computation.entity.private_computation_instance`. from fbpcs.private_computation.entity.private_computation_instance import ( PrivateComputationRole, ) + +# pyre-fixme[21]: Could not find module +# `fbpcs.private_computation.service.pc_pre_validation_stage_service`. from fbpcs.private_computation.service.pc_pre_validation_stage_service import ( PRE_VALIDATION_CHECKS_TIMEOUT, ) + +# pyre-fixme[21]: Could not find module +# `fbpcs.private_computation.service.pre_validation_util`. from fbpcs.private_computation.service.pre_validation_util import get_cmd_args + +# pyre-fixme[21]: Could not find module +# `fbpcs.private_computation.service.private_computation`. from fbpcs.private_computation.service.private_computation import ( PrivateComputationService, ) + +# pyre-fixme[21]: Could not find module +# `fbpcs.private_computation.service.run_binary_base_service`. from fbpcs.private_computation.service.run_binary_base_service import ( RunBinaryBaseService, ) + +# pyre-fixme[21]: Could not find module `fbpcs.private_computation.service.utils`. from fbpcs.private_computation.service.utils import generate_env_vars_dict + +# pyre-fixme[21]: Could not find module +# `fbpcs.private_computation_cli.private_computation_service_wrapper`. from fbpcs.private_computation_cli.private_computation_service_wrapper import ( build_private_computation_service, ) @@ -35,14 +60,18 @@ class PreValidateService: @staticmethod async def run_pre_validate_async( + # pyre-fixme[11]: Annotation `PrivateComputationService` is not defined as a + # type. pc_service: PrivateComputationService, input_paths: List[str], logger: logging.Logger, ) -> None: region = pc_service.pc_validator_config.region onedocker_svc = pc_service.onedocker_svc + # pyre-fixme[16]: Module `fbpcs` has no attribute `onedocker_binary_names`. binary_name = OneDockerBinaryNames.PC_PRE_VALIDATION.value binary_config = pc_service.onedocker_binary_config_map[binary_name] + # pyre-fixme[16]: Module `service` has no attribute `utils`. env_vars = generate_env_vars_dict(repository_path=binary_config.repository_path) """ @@ -50,6 +79,7 @@ async def run_pre_validate_async( Since this service runs on partner PCE, we are hardcoding PrivateComputationRole to PARTNER in the code below. """ cmd_args = [ + # pyre-fixme[16]: Module `service` has no attribute `pre_validation_util`. get_cmd_args( input_path=input_path, region=region, @@ -57,6 +87,8 @@ async def run_pre_validate_async( pre_validation_file_stream_flag=True, publisher_pc_pre_validation_flag=True, partner_pc_pre_validation_flag=True, + # pyre-fixme[16]: Module `entity` has no attribute + # `private_computation_instance`. private_computation_role=PrivateComputationRole.PARTNER, input_path_start_ts=None, input_path_end_ts=None, @@ -64,17 +96,21 @@ async def run_pre_validate_async( for input_path in input_paths ] + # pyre-fixme[16]: Module `service` has no attribute `run_binary_base_service`. container_instances = await RunBinaryBaseService().start_containers( cmd_args, onedocker_svc, binary_config.binary_version, binary_name, + # pyre-fixme[16]: Module `service` has no attribute + # `pc_pre_validation_stage_service`. timeout=PRE_VALIDATION_CHECKS_TIMEOUT, env_vars=env_vars, container_type=ContainerType.LARGE, ) logger.info("Started container instances") + # pyre-fixme[16]: Module `service` has no attribute `run_binary_base_service`. completed_containers = await RunBinaryBaseService().wait_for_containers_async( onedocker_svc, container_instances, @@ -119,6 +155,7 @@ def pre_validate( input_paths: List[str], logger: logging.Logger, ) -> None: + # pyre-fixme[16]: Module `fbpcs` has no attribute `private_computation_cli`. pc_service = build_private_computation_service( config["private_computation"], config["mpc"], diff --git a/fbpcs/private_computation/service/run_binary_base_service.py b/fbpcs/private_computation/service/run_binary_base_service.py index 169d8df36..479a88c01 100644 --- a/fbpcs/private_computation/service/run_binary_base_service.py +++ b/fbpcs/private_computation/service/run_binary_base_service.py @@ -10,13 +10,25 @@ import logging from typing import Dict, List, Optional +# pyre-fixme[21]: Could not find module `fbpcp.entity.certificate_request`. from fbpcp.entity.certificate_request import CertificateRequest +# pyre-fixme[21]: Could not find module `fbpcp.entity.container_instance`. from fbpcp.entity.container_instance import ContainerInstance, ContainerInstanceStatus + +# pyre-fixme[21]: Could not find module `fbpcp.entity.container_permission`. from fbpcp.entity.container_permission import ContainerPermissionConfig + +# pyre-fixme[21]: Could not find module `fbpcp.entity.container_type`. from fbpcp.entity.container_type import ContainerType + +# pyre-fixme[21]: Could not find module `fbpcp.error.pcp`. from fbpcp.error.pcp import ThrottlingError + +# pyre-fixme[21]: Could not find module `fbpcp.service.onedocker`. from fbpcp.service.onedocker import OneDockerService + +# pyre-fixme[21]: Could not find module `fbpcs.common.service.retry_handler`. from fbpcs.common.service.retry_handler import BackoffType, RetryHandler from fbpcs.private_computation.service.constants import DEFAULT_CONTAINER_TIMEOUT_IN_SEC @@ -27,6 +39,7 @@ class RunBinaryBaseService: async def start_containers( self, cmd_args_list: List[str], + # pyre-fixme[11]: Annotation `OneDockerService` is not defined as a type. onedocker_svc: OneDockerService, binary_version: str, binary_name: str, @@ -34,11 +47,16 @@ async def start_containers( wait_for_containers_to_finish: bool = False, env_vars: Optional[Dict[str, str]] = None, wait_for_containers_to_start_up: bool = True, + # pyre-fixme[11]: Annotation `ContainerInstance` is not defined as a type. existing_containers: Optional[List[ContainerInstance]] = None, + # pyre-fixme[11]: Annotation `ContainerType` is not defined as a type. container_type: Optional[ContainerType] = None, + # pyre-fixme[11]: Annotation `CertificateRequest` is not defined as a type. certificate_request: Optional[CertificateRequest] = None, env_vars_list: Optional[List[Dict[str, str]]] = None, opa_workflow_path: Optional[str] = None, + # pyre-fixme[11]: Annotation `ContainerPermissionConfig` is not defined as a + # type. permission: Optional[ContainerPermissionConfig] = None, ) -> List[ContainerInstance]: logger = logging.getLogger(__name__) @@ -82,6 +100,7 @@ async def start_containers( logger.info("Skipped container warm up") return pending_containers + # pyre-fixme[16]: Module `fbpcs` has no attribute `common`. with RetryHandler( ThrottlingError, logger=logger, backoff_seconds=30 ) as retry_handler: @@ -150,8 +169,12 @@ async def wait_for_containers_async( container_ids = [container.instance_id for container in containers] finished_containers = [] + # pyre-fixme[16]: Module `fbpcs` has no attribute `common`. with RetryHandler( - ThrottlingError, backoff_type=BackoffType.LINEAR + # pyre-fixme[16]: Module `fbpcs` has no attribute `common`. + ThrottlingError, + # pyre-fixme[16]: Module `fbpcs` has no attribute `common`. + backoff_type=BackoffType.LINEAR, ) as retry_handler: updated_containers = retry_handler.execute_sync( onedocker_svc.get_containers, @@ -164,8 +187,12 @@ async def wait_for_containers_async( ) while pending_container_ids: await asyncio.sleep(poll) + # pyre-fixme[16]: Module `fbpcs` has no attribute `common`. with RetryHandler( - ThrottlingError, backoff_type=BackoffType.LINEAR + # pyre-fixme[16]: Module `fbpcs` has no attribute `common`. + ThrottlingError, + # pyre-fixme[16]: Module `fbpcs` has no attribute `common`. + backoff_type=BackoffType.LINEAR, ) as retry_handler: updated_containers = retry_handler.execute_sync( onedocker_svc.get_containers, diff --git a/fbpcs/service/workflow_sfn_fb.py b/fbpcs/service/workflow_sfn_fb.py index 8775bdeed..573a6fb0d 100644 --- a/fbpcs/service/workflow_sfn_fb.py +++ b/fbpcs/service/workflow_sfn_fb.py @@ -10,10 +10,12 @@ import botocore +# pyre-fixme[21]: Could not find module `fbpcp.intern.gateway.aws_fb`. from fbpcp.intern.gateway.aws_fb import FBAWSGateway from fbpcs.service.workflow_sfn import SfnWorkflowService +# pyre-fixme[11]: Annotation `FBAWSGateway` is not defined as a type. class FBSfnWorkflowService(FBAWSGateway, SfnWorkflowService): def __init__( self, @@ -21,7 +23,9 @@ def __init__( account: str, role: Optional[str] = None, ) -> None: + # pyre-fixme[28]: Unexpected keyword argument `account`. super().__init__(account=account, role=role, region=region) + # pyre-fixme[16]: `FBSfnWorkflowService` has no attribute `session_gen`. self.client: botocore.client.BaseClient = self.session_gen.get_client( "stepfunctions" )