B&R for UDP encryption binary (facebookresearch#2200)

Summary:
Pull Request resolved: facebookresearch#2200

As title

Differential Revision: D44179453

fbshipit-source-id: 5bfe6cdf7d3413c743ac04618d5850bef5a5c13d

binaries_out_lists/emp_games.txt

@@ -8,3 +8,4 @@ pcf2_attribution_calculator
 pcf2_aggregation_calculator
 shard_aggregator
 private_id_dfca_aggregator
+udp_encryptor

docker/emp_games/CMakeLists.txt

@@ -3,201 +3,43 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
-cmake_minimum_required(VERSION 3.10.1)
+cmake_minimum_required(VERSION 3.16)
 project(game)
 
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
+add_library(range_v3 INTERFACE IMPORTED)
+set_target_properties(range_v3 PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES /root/build/range-v3/include)
+find_program(CMAKE_CXX_COMPILER clang++ PATHS CACHE STRING "C++ compiler" FORCE)
+
 # Don't compile with AVX512 instructions since many of the AWS
 # instances won't have access to that instruction set.
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mno-avx512f -march=haswell")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mno-avx512f -march=haswell")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mno-avx512f -march=haswell -fcoroutines")
 
 include("common.cmake")
 include("perf_tools.cmake")
 
-# lift
-add_executable(
-  lift_calculator
-  "fbpcs/emp_games/lift/calculator/main.cpp"
-  "fbpcs/emp_games/lift/calculator/OutputMetrics.hpp"
-  "fbpcs/emp_games/lift/common/GroupedLiftMetrics.h"
-  "fbpcs/emp_games/lift/common/GroupedLiftMetrics.cpp"
-  "fbpcs/emp_games/lift/common/LiftMetrics.h"
-  "fbpcs/emp_games/lift/common/LiftMetrics.cpp"
-  "fbpcs/emp_games/lift/calculator/CalculatorApp.h"
-  "fbpcs/emp_games/lift/calculator/CalculatorApp.cpp"
-  "fbpcs/emp_games/lift/calculator/CalculatorGame.h"
-  "fbpcs/emp_games/lift/calculator/OutputMetrics.h"
-  "fbpcs/emp_games/lift/calculator/InputData.cpp"
-  "fbpcs/emp_games/lift/calculator/InputData.h"
-  "fbpcs/emp_games/lift/calculator/CalculatorGameConfig.h"
-  "fbpcs/emp_games/lift/calculator/OutputMetricsData.h"
-  "fbpcs/emp_games/common/PrivateData.h"
-  "fbpcs/emp_games/common/SecretSharing.h"
-  "fbpcs/emp_games/common/EmpOperationUtil.h"
-  "fbpcs/emp_games/common/Csv.h"
-  "fbpcs/emp_games/common/Csv.cpp")
-target_link_libraries(
-  lift_calculator
-  empgamecommon)
-install(TARGETS lift_calculator DESTINATION bin)
-
-# generic shard_aggregator
-file(GLOB shard_aggregator_src
-  "fbpcs/emp_games/attribution/shard_aggregator/AggMetrics.cpp",
-  "fbpcs/emp_games/attribution/shard_aggregator/AggMetricsThresholdCheckers.cpp",
-  "fbpcs/emp_games/attribution/shard_aggregator/ShardAggregatorApp.cpp",
-  "fbpcs/emp_games/attribution/shard_aggregator/ShardAggregatorValidation.cpp",
-  "fbpcs/emp_games/attribution/shard_aggregator/main.cpp"
-  "fbpcs/emp_games/attribution/Aggregator.h"
-  "fbpcs/emp_games/attribution/AttributionMetrics.h"
-  "fbpcs/emp_games/attribution/AttributionRule.h"
-  "fbpcs/emp_games/attribution/Constants.h"
-  "fbpcs/emp_games/attribution/Conversion.h"
-  "fbpcs/emp_games/attribution/Debug.h"
-  "fbpcs/emp_games/attribution/Touchpoint.h",
-  "fbpcs/emp_games/attribution/shard_aggregator/MainUtil.h",
-  "fbpcs/emp_games/attribution/shard_aggregator/ShardAggregatorGame.h")
-add_executable(
-  shard_aggregator
-  ${shard_aggregator_src})
-target_link_libraries(
-  shard_aggregator
-  empgamecommon
-  perftools)
-install(TARGETS shard_aggregator DESTINATION bin)
-
-# decoupled_attribution
-file(GLOB decoupled_attribution_src
-  "fbpcs/emp_games/attribution/decoupled_attribution/**.c"
-  "fbpcs/emp_games/attribution/decoupled_attribution/**.cpp"
-  "fbpcs/emp_games/attribution/decoupled_attribution/**.h"
-  "fbpcs/emp_games/attribution/decoupled_attribution/**.hpp")
-list(FILTER decoupled_attribution_src EXCLUDE REGEX ".*Test.*")
-add_executable(
-  decoupled_attribution_calculator
-  ${decoupled_attribution_src})
-target_link_libraries(
-  decoupled_attribution_calculator
-  empgamecommon
-  perftools)
-install(TARGETS decoupled_attribution_calculator DESTINATION bin)
-
-# decoupled_aggregation
-file(GLOB decoupled_aggregation_src
-  "fbpcs/emp_games/attribution/decoupled_aggregation/**.c"
-  "fbpcs/emp_games/attribution/decoupled_aggregation/**.cpp"
-  "fbpcs/emp_games/attribution/decoupled_aggregation/**.h"
-  "fbpcs/emp_games/attribution/decoupled_aggregation/metadata/**.h"
-  "fbpcs/emp_games/attribution/decoupled_aggregation/**.hpp")
-list(FILTER decoupled_aggregation_src EXCLUDE REGEX ".*Test.*")
-add_executable(
-  decoupled_aggregation_calculator
-  ${decoupled_aggregation_src})
-target_link_libraries(
-  decoupled_aggregation_calculator
-  empgamecommon
-  perftools)
-install(TARGETS decoupled_aggregation_calculator DESTINATION bin)
-
-
-# Find boost
-find_package(Boost REQUIRED)
-include_directories(${Boost_INCLUDE_DIRS})
-
-# pcf2_attribution
-file(GLOB pcf2_attribution_src
-  "fbpcs/emp_games/pcf2_attribution/**.c"
-  "fbpcs/emp_games/pcf2_attribution/**.cpp"
-  "fbpcs/emp_games/pcf2_attribution/**.h"
-  "fbpcs/emp_games/pcf2_attribution/**.hpp")
-list(FILTER pcf2_attribution_src EXCLUDE REGEX ".*Test.*")
-add_executable(
-  pcf2_attribution_calculator
-  ${pcf2_attribution_src})
-target_link_libraries(
-  pcf2_attribution_calculator
-  empgamecommon
-  perftools)
-install(TARGETS pcf2_attribution_calculator DESTINATION bin)
-
-# pcf2_aggregation
-file(GLOB pcf2_aggregation_src
-  "fbpcs/emp_games/pcf2_aggregation/**.c"
-  "fbpcs/emp_games/pcf2_aggregation/**.cpp"
-  "fbpcs/emp_games/pcf2_aggregation/**.h"
-  "fbpcs/emp_games/pcf2_aggregation/**.hpp")
-list(FILTER pcf2_aggregation_src EXCLUDE REGEX ".*Test.*")
-add_executable(
-  pcf2_aggregation_calculator
-  ${pcf2_aggregation_src})
-target_link_libraries(
-  pcf2_aggregation_calculator
-  empgamecommon
-  perftools)
-install(TARGETS pcf2_aggregation_calculator DESTINATION bin)
+message(STATUS "CMAKE_CXX_STANDARD: ${CMAKE_CXX_STANDARD}")
+message(STATUS "CMAKE_CXX_COMPILER: ${CMAKE_CXX_COMPILER}")
+message(STATUS "CMAKE_CXX_FLAGS: ${CMAKE_CXX_FLAGS}")
+message(STATUS "CMAKE_LINKER: ${CMAKE_LINKER}")
 
-# pcf2_lift metadata compaction
-file(GLOB pcf2_lift_metadata_compaction_src
-  "fbpcs/emp_games/lift/metadata_compaction/**.cpp"
-  "fbpcs/emp_games/lift/metadata_compaction/**.h")
-list(FILTER pcf2_lift_metadata_compaction_src EXCLUDE REGEX ".*Test.*")
+# pcf2_udp encryption
+file(GLOB udp_encryptor_src
+  "fbpcs/emp_games/data_processing/unified_data_process/UdpEncryptor/**.cpp"
+  "fbpcs/emp_games/data_processing/unified_data_process/UdpEncryptor/**.h")
+list(FILTER udp_encryptor_src EXCLUDE REGEX ".*Test.*")
 add_executable(
-  pcf2_lift_metadata_compaction
-  ${pcf2_lift_metadata_compaction_src})
+  udp_encryptor
+  ${udp_encryptor_src})
 target_link_libraries(
-  pcf2_lift_metadata_compaction
+  udp_encryptor
   empgamecommon
   perftools
-  pcf2_lift_input_processing
+  ${FOLLY_LIBRARIES}
   ${Boost_LIBRARIES}
+  range_v3
 )
-install(TARGETS pcf2_lift_metadata_compaction DESTINATION bin)
-
-# pcf2_lift
-file(GLOB pcf2_lift_calculator_src
-  "fbpcs/emp_games/lift/common/**.cpp"
-  "fbpcs/emp_games/lift/common/**.h"
-  "fbpcs/emp_games/lift/pcf2_calculator/**.cpp"
-  "fbpcs/emp_games/lift/pcf2_calculator/**.h")
-list(FILTER pcf2_lift_calculator_src EXCLUDE REGEX ".*Test.*")
-add_executable(
-  pcf2_lift_calculator
-  ${pcf2_lift_calculator_src})
-target_link_libraries(
-  pcf2_lift_calculator
-  empgamecommon
-  perftools
-  pcf2_lift_input_processing)
-install(TARGETS pcf2_lift_calculator DESTINATION bin)
-
-# pcf2_shard_combiner
-file(GLOB pcf2_shard_combiner_src
-  "fbpcs/emp_games/pcf2_shard_combiner/**.cpp"
-  "fbpcs/emp_games/pcf2_shard_combiner/**.h"
-  "fbpcs/emp_games/pcf2_shard_combiner/util/**.h")
-list(FILTER pcf2_shard_combiner_src EXCLUDE REGEX ".*Test.*")
-add_executable(
-  pcf2_shard_combiner
-  ${pcf2_shard_combiner_src})
-target_link_libraries(
-  pcf2_shard_combiner
-  empgamecommon
-  perftools)
-install(TARGETS pcf2_shard_combiner DESTINATION bin)
-
-# private_id_dfca_aggregator
-file(GLOB private_id_dfca_aggregator_src
-  "fbpcs/emp_games/private_id_dfca_aggregator/**.cpp"
-  "fbpcs/emp_games/private_id_dfca_aggregator/**.h"
-  "fbpcs/emp_games/private_id_dfca_aggregator/util/**.h")
-list(FILTER private_id_dfca_aggregator_src EXCLUDE REGEX ".*Test.*")
-add_executable(
-  private_id_dfca_aggregator
-  ${private_id_dfca_aggregator_src})
-target_link_libraries(
-  private_id_dfca_aggregator
-  empgamecommon
-  perftools)
-install(TARGETS private_id_dfca_aggregator DESTINATION bin)
+install(TARGETS udp_encryptor DESTINATION bin)

docker/emp_games/Dockerfile.ubuntu

@@ -7,6 +7,11 @@ ARG fbpcf_image
 
 FROM ${fbpcf_image} as dev
 
+WORKDIR /root/build
+RUN git clone https://github.com/ericniebler/range-v3
+
+
+
 RUN mkdir -p /root/build/emp_game
 WORKDIR /root/build/emp_game
 
@@ -25,6 +30,9 @@ COPY fbpcs/emp_games/pcf2_shard_combiner/ ./fbpcs/emp_games/pcf2_shard_combiner
 COPY fbpcs/emp_games/private_id_dfca_aggregator/ ./fbpcs/emp_games/private_id_dfca_aggregator
 COPY fbpcs/emp_games/lift/ ./fbpcs/emp_games/lift
 COPY fbpcs/emp_games/common/ ./fbpcs/emp_games/common
+COPY fbpcs/emp_games/data_processing/global_parameters ./fbpcs/emp_games/data_processing/global_parameters
+COPY fbpcs/emp_games/data_processing/unified_data_process/UdpEncryptor ./fbpcs/emp_games/data_processing/unified_data_process/UdpEncryptor
+COPY fbpcs/emp_games/data_processing/unified_data_process/UdpDecryptor ./fbpcs/emp_games/data_processing/unified_data_process/UdpDecryptor
 
 RUN cmake . -DTHREADING=ON -DUSE_RANDOM_DEVICE=ON
 RUN ./make_and_install_binary.sh
@@ -45,7 +53,8 @@ RUN apt-get -y update && apt-get install -y --no-install-recommends \
     libgoogle-glog0v5 \
     libssl1.1 \
     libre2-5 \
-    zlib1g
+    zlib1g \
+    librange-v3-dev
 
 COPY --from=dev /root/build/emp_game/bin/. /usr/local/bin/.
 RUN useradd -ms /bin/bash pcs

docker/emp_games/common.cmake

@@ -18,7 +18,10 @@ file(GLOB emp_game_common_src
   "fbpcs/emp_games/common/**.c"
   "fbpcs/emp_games/common/**.cpp"
   "fbpcs/emp_games/common/**.h"
-  "fbpcs/emp_games/common/**.hpp")
+  "fbpcs/emp_games/common/**.hpp"
+  "fbpcs/emp_games/data_processing/global_parameters/**.cpp"
+  "fbpcs/emp_games/data_processing/global_parameters/**.h"
+  )
 list(FILTER emp_game_common_src EXCLUDE REGEX ".*Test.*")
 add_library(empgamecommon STATIC
   ${emp_game_common_src})
@@ -30,7 +33,7 @@ target_link_libraries(
   ${AWSSDK_LINK_LIBRARIES}
   ${EMP-OT_LIBRARIES}
   google-cloud-cpp::storage
-  Folly::folly
+  ${FOLLY_LIBRARIES}
   re2)
 
 # pcf2 lift input processing

extract-docker-binaries.sh

@@ -68,6 +68,7 @@ docker create -ti --name temp_container "${DOCKER_IMAGE_PATH}"
 docker cp temp_container:/usr/local/bin/lift_calculator "$SCRIPT_DIR/binaries_out/."
 docker cp temp_container:/usr/local/bin/pcf2_lift_calculator "$SCRIPT_DIR/binaries_out/."
 docker cp temp_container:/usr/local/bin/pcf2_lift_metadata_compaction "$SCRIPT_DIR/binaries_out/."
+docker cp temp_container:/usr/local/bin/udp_encryptor "$SCRIPT_DIR/binaries_out/."
 docker cp temp_container:/usr/local/bin/decoupled_attribution_calculator "$SCRIPT_DIR/binaries_out/."
 docker cp temp_container:/usr/local/bin/decoupled_aggregation_calculator "$SCRIPT_DIR/binaries_out/."
 docker cp temp_container:/usr/local/bin/pcf2_attribution_calculator "$SCRIPT_DIR/binaries_out/."

promote_scripts/promote_binaries.sh

@@ -42,6 +42,7 @@ binary_names=(
     'data_processing/private_id_dfca_id_combiner'
     'validation/pc_pre_validation_cli'
     'smart_agent/smart_agent_server'
+    'udp_encryptor'
 )
 
 s3_path="s3://one-docker-repository-prod"

upload-binaries-to-s3-test.sh

@@ -46,6 +46,7 @@ data_processing_repo="s3://$one_docker_repo/data_processing"
 private_id_repo="s3://$one_docker_repo/pid"
 validation_repo="s3://$one_docker_repo/validation"
 smart_agent_repo="s3://$one_docker_repo/smart_agent"
+udp_encryptor_package="s3://$one_docker_repo/data_processing/unified_data_process/UdpEncryptor/${TAG}/udp_encryptor"
 
 if [ "$PACKAGE" = "emp_games" ]; then
 cd binaries_out || exit
@@ -59,6 +60,7 @@ aws s3 cp pcf2_aggregation_calculator "$pcf2_aggregation"
 aws s3 cp shard_aggregator "$shard_aggregator_package"
 aws s3 cp pcf2_shard_combiner "$pcf2_shard_combiner_package"
 aws s3 cp private_id_dfca_aggregator "$private_id_dfca_aggregator_package"
+aws s3 cp udp_encryptor "$udp_encryptor_package"
 cd .. || exit
 fi
 

upload_scripts/upload-binaries-using-onedocker.sh

@@ -63,6 +63,7 @@ pcf2_aggregation_path="private_attribution/pcf2_aggregation"
 shard_aggregator_path="private_attribution/shard-aggregator"
 pcf2_shard_combiner_path="private_attribution/pcf2_shard-combiner"
 private_id_dfca_aggregator_path="private_id_dfca/private_id_dfca_aggregator"
+udp_encryptor_path="data_processing/unified_data_process/UdpEncryptor"
 
 if [ "$PACKAGE" = "emp_games" ]; then
 cd binaries_out || exit
@@ -76,6 +77,7 @@ onedocker_upload "$pcf2_aggregation_path" pcf2_aggregation_calculator
 onedocker_upload "$shard_aggregator_path" shard_aggregator
 onedocker_upload "$pcf2_shard_combiner_path" pcf2_shard_combiner
 onedocker_upload "$private_id_dfca_aggregator_path" private_id_dfca_aggregator
+onedocker_upload "$udp_encryptor_path" udp_encryptor
 cd .. || exit
 fi