Merge pull request #2 from Worklytics/s146-improve-init

improve initialization, update to 0.4.18

Author: eschultink

README.md

@@ -17,6 +17,9 @@ You'll need:
   - a Bash-like shell environment on Linux, MacOS, or [WSL on Windows](https://learn.microsoft.com/en-us/windows/wsl/install).
   - [`git` installed](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git), although it
     is usually included on those platforms (check with `git --version`).
+  - an AWS account and credentials, as described in [Psoxy's AWS - Getting Started docs](https://github.com/Worklytics/psoxy/blob/v0.4.18/docs/aws/getting-started.md)
+  - the [prerequisites for Psoxy](https://github.com/Worklytics/psoxy/blob/v0.4.18/README.md#prerequisites)
+    itself, although this example will attempt to help you check those
 
 ### Setup
 
@@ -39,22 +42,28 @@ git clone https://github.com/{{YOUR_ORG_ID}}/{{YOUR_REPO_NAME}}.git
 ./check-prereqs
 ```
 
- 4. Initialize your configuration
+ 4. Authenticate your tools as needed:
+
+   - [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html) - `aws get-caller-identity` should work and return your expected account/user
+   - if plan to get data from Google Workspace, auth [GCloud CLI](https://cloud.google.com/sdk/docs/authorizing) - `gcloud auth login` to authenticate, then `gcloud auth list` to verify you have expected account/user
+   - if plan to get data from Microsoft 365, auth [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli) - `az login --allow-no-subscription` to authenticate, then `az account list` to verify you have expected account/user
+
+ 5. Initialize your configuration
 
 ```shell
 ./init
 ```
 
- 5. Review your `terraform.tfvars` file; customize as needed (eg, comment out datasources you don't need).
+ 6. Review your `terraform.tfvars` file; customize as needed (eg, comment out datasources you don't need).
 
- 6. Run `terraform plan` and review results to understand what will be created. Customize your
+ 7. Run `terraform plan` and review results to understand what will be created. Customize your
     `terraform.tfvars` or `main.tf` file if needed.
 
 ```shell
 terraform plan
 ```
 
- 7. Run `terraform apply` to create the resources.
+ 8. Run `terraform apply` to create the resources.
 ```shell
 terraform apply
 ```

build

@@ -17,6 +17,12 @@ if ! git --version &> /dev/null ; then
   exit 1
 fi
 
+if ! terraform -v &> /dev/null ; then
+  printf "${RED}Terraform CLI not available.${NC} Psoxy examples / deployment scripts require it. See https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli\n"
+  if $HOMEBREW_AVAILABLE; then printf " or, as you have Homebrew available, run ${BLUE}brew install terraform${NC}\n"; fi
+  exit 1
+fi
+
 if ! mvn -v &> /dev/null ; then
   printf "${RED}Maven not installed.${NC} See https://maven.apache.org/install.html\n"
   if $HOMEBREW_AVAILABLE; then printf " or, as you have Homebrew available, run ${BLUE}brew install maven${NC}\n"; fi
@@ -62,7 +68,7 @@ if ! gcloud --version &> /dev/null ; then
   printf "${RED}Google Cloud SDK is not installed.${NC} ${GCLOUD_REASON} See https://cloud.google.com/sdk/docs/install\n"
   if $HOMEBREW_AVAILABLE; then printf " or, as you have Homebrew available, run ${BLUE}brew install --cask google-cloud-sdk${NC}\n"; fi
 else
-  printf "Google Cloud SDK version ${BLUE}`gcloud --version | head -n 1`${NC} is installed.\n"
+  printf "Google Cloud SDK version ${BLUE}`gcloud --version 2> /dev/null | head -n 1`${NC} is installed.\n"
   printf "\t- make sure ${BLUE}gcloud auth list --filter=\"status:ACTIVE\"${NC} returns the account you expect. $GCLOUD_REASON\n"
 fi
 
@@ -75,7 +81,5 @@ if ! az --version &> /dev/null ; then
 else
   # how can pipe to sed or something to strip extra whitespace out?
   printf "Azure CLI version ${BLUE}`az --version --only-show-errors | head -n 1`${NC} is installed.\n"
-  printf "\t- make sure ${BLUE}az account show${NC} is the user/tenant you expect. $AZCLI_REASON\n"
+  printf "\t- make sure ${BLUE}az account show${NC} is the user/tenant you expect. If not, ${BLUE}az login --allow-no-subscription${NC} to authenticate. $AZCLI_REASON\n"
 fi
-
-# TODO: check auth aws, gcp, azure ??

check-prereqs

@@ -56,8 +56,8 @@ provider "azuread" {
 }
 
 module "psoxy" {
-  # source = "../psoxy/infra/modular-examples/aws"
-  source = "git::https://github.com/worklytics/psoxy//infra/modular-examples/aws?ref=v0.4.17"
+  # source = "../../modular-examples/aws"
+  source = "git::https://github.com/worklytics/psoxy//infra/modular-examples/aws?ref=v0.4.18"
 
   aws_account_id                 = var.aws_account_id
   aws_assume_role_arn            = var.aws_assume_role_arn # role that can test the instances (lambdas)
@@ -71,6 +71,7 @@ module "psoxy" {
   non_production_connectors      = var.non_production_connectors
   connector_display_name_suffix  = var.connector_display_name_suffix
   custom_bulk_connectors         = var.custom_bulk_connectors
+  custom_rest_rules              = var.custom_rest_rules
   lookup_table_builders          = var.lookup_table_builders
   msft_tenant_id                 = var.msft_tenant_id
   msft_owners_email              = var.msft_owners_email

main.tf

@@ -130,22 +130,12 @@ variable "general_environment_variables" {
 variable "pseudonymize_app_ids" {
   type        = string
   description = "if set, will set value of PSEUDONYMIZE_APP_IDS environment variable to this value for all sources"
-  default     = false
+  default     = true
 }
 
 variable "enabled_connectors" {
   type        = list(string)
   description = "list of ids of connectors to enabled; see modules/worklytics-connector-specs"
-
-  default = [
-    "azure-ad",
-    "outlook-cal",
-    "outlook-mail",
-    "asana",
-    "hris",
-    "slack-discovery-api",
-    "zoom",
-  ]
 }
 
 variable "non_production_connectors" {
@@ -156,14 +146,20 @@ variable "non_production_connectors" {
 
 variable "bulk_input_expiration_days" {
   type        = number
-  description = "**alpha** Number of days after which objects in the bucket will expire. This could be as low as 1 day; longer aids debugging of issues."
+  description = "Number of days after which objects in the bucket will expire. This could be as low as 1 day; longer aids debugging of issues."
   default     = 30
 }
 
 variable "bulk_sanitized_expiration_days" {
   type        = number
-  description = "**alpha** Number of days after which objects in the bucket will expire. In practice, Worklytics syncs data ~weekly, so 30 day minimum for this value."
-  default     = 720
+  description = "Number of days after which objects in the bucket will expire. In practice, Worklytics syncs data ~weekly, so 30 day minimum for this value."
+  default     = 1805 # 5 years; intent is 'forever', but some upperbound in case bucket is forgotten
+}
+
+variable "custom_rest_rules" {
+  type        = map(string)
+  description = "map of connector id --> YAML file with custom rules"
+  default     = {}
 }
 
 variable "custom_bulk_connectors" {
@@ -235,4 +231,3 @@ variable "lookup_table_builders" {
     #    }
   }
 }
-