diff --git a/.github/Architecture.md b/.github/Architecture.md index c140163d..dc8ac020 100644 --- a/.github/Architecture.md +++ b/.github/Architecture.md @@ -5,61 +5,61 @@ ``` 📦 training | |- 📂 training: +| | |- 📂 middleware: +| | | |- 📜 __init__.py +| | | |- 📜 health_check_middleware.py | | |- 📂 routes: -| | | |- 📂 tabular: -| | | | |- 📜 tabular.py +| | | |- 📂 datasets: +| | | | |- 📂 default: +| | | | | |- 📜 schemas.py +| | | | | |- 📜 __init__.py +| | | | | |- 📜 columns.py | | | | |- 📜 __init__.py -| | | | |- 📜 schemas.py | | | |- 📂 image: +| | | | |- 📜 schemas.py | | | | |- 📜 __init__.py | | | | |- 📜 image.py -| | | | |- 📜 schemas.py | | | |- 📂 audio: +| | | | |- 📜 schemas.py | | | | |- 📜 __init__.py | | | | |- 📜 audio.py +| | | |- 📂 tabular: | | | | |- 📜 schemas.py -| | | |- 📂 datasets: -| | | | |- 📂 default: -| | | | | |- 📜 columns.py -| | | | | |- 📜 __init__.py -| | | | | |- 📜 schemas.py | | | | |- 📜 __init__.py -| | | |- 📜 __init__.py +| | | | |- 📜 tabular.py | | | |- 📜 schemas.py -| | |- 📂 middleware: | | | |- 📜 __init__.py -| | | |- 📜 health_check_middleware.py | | |- 📂 core: | | | |- 📂 celery: +| | | | |- 📜 Dockerfile +| | | | |- 📜 __init__.py +| | | | |- 📜 optimizer.py : what optimizer to use (ie: SGD or Adam for now) +| | | | |- 📜 dataset.py : read in the dataset through URL or file upload +| | | | |- 📜 train_types.py | | | | |- 📜 trainer.py -| | | | |- 📜 criterion.py | | | | |- 📜 dl_model.py : torch model based on user specifications from drag and drop -| | | | |- 📜 train_types.py -| | | | |- 📜 dataset.py : read in the dataset through URL or file upload -| | | | |- 📜 __init__.py -| | | | |- 📜 Dockerfile +| | | | |- 📜 criterion.py | | | | |- 📜 worker.py -| | | | |- 📜 optimizer.py : what optimizer to use (ie: SGD or Adam for now) | | | |- 📜 __init__.py | | | |- 📜 authenticator.py -| | |- 📜 asgi.py -| | |- 📜 constants.py : list of helpful constants -| | |- 📜 celery_app.py -| | |- 📜 settings.py | | |- 📜 __init__.py | | |- 📜 wsgi.py -| | |- 📜 urls.py | | |- 📜 celeryconfig.py +| | |- 📜 settings.py +| | |- 📜 asgi.py +| | |- 📜 urls.py +| | |- 📜 celery_app.py +| | |- 📜 constants.py : list of helpful constants +| |- 📜 pytest.ini +| |- 📜 cli.py +| |- 📜 Dockerfile | |- 📜 README.md | |- 📜 docker-compose.yml -| |- 📜 cli.py +| |- 📜 docker-compose.prod.yml | |- 📜 pyproject.toml +| |- 📜 environment.yml | |- 📜 poetry.lock -| |- 📜 pytest.ini -| |- 📜 Dockerfile | |- 📜 manage.py -| |- 📜 environment.yml -| |- 📜 docker-compose.prod.yml ``` ## Frontend Architecture @@ -68,179 +68,179 @@ 📦 frontend | |- 📂 public: | | |- 📂 images: +| | | |- 📂 learn_mod_images: +| | | | |- 📜 neuralnet.png +| | | | |- 📜 ReLUactivation.png +| | | | |- 📜 binarystepactivation.png +| | | | |- 📜 neuron.png +| | | | |- 📜 sigmoidactivation.png +| | | | |- 📜 robotImage.jpg +| | | | |- 📜 LeakyReLUactivation.png +| | | | |- 📜 neuronWithEquation.png +| | | | |- 📜 lossExampleTable.png +| | | | |- 📜 lossExampleEquation.png +| | | | |- 📜 sigmoidfunction.png +| | | | |- 📜 tanhactivation.png +| | | | |- 📜 lossExample.png | | | |- 📂 logos: | | | | |- 📂 dlp_branding: -| | | | | |- 📜 dlp-logo.png : DLP Logo, duplicate of files in public, but essential as the frontend can't read public | | | | | |- 📜 dlp-logo.svg : DLP Logo, duplicate of files in public, but essential as the frontend can't read public -| | | | |- 📜 dsgt-logo-white-back.png +| | | | | |- 📜 dlp-logo.png : DLP Logo, duplicate of files in public, but essential as the frontend can't read public +| | | | |- 📜 dsgt-logo-light.png +| | | | |- 📜 pytorch-logo.png | | | | |- 📜 python-logo.png | | | | |- 📜 google.png -| | | | |- 📜 pandas-logo.png -| | | | |- 📜 react-logo.png -| | | | |- 📜 flask-logo.png | | | | |- 📜 aws-logo.png | | | | |- 📜 github.png +| | | | |- 📜 flask-logo.png +| | | | |- 📜 dsgt-logo-white-back.png | | | | |- 📜 dsgt-logo-dark.png -| | | | |- 📜 dsgt-logo-light.png -| | | | |- 📜 pytorch-logo.png -| | | |- 📂 learn_mod_images: -| | | | |- 📜 neuron.png -| | | | |- 📜 ReLUactivation.png -| | | | |- 📜 LeakyReLUactivation.png -| | | | |- 📜 lossExampleEquation.png -| | | | |- 📜 lossExampleTable.png -| | | | |- 📜 robotImage.jpg -| | | | |- 📜 neuralnet.png -| | | | |- 📜 sigmoidfunction.png -| | | | |- 📜 lossExample.png -| | | | |- 📜 tanhactivation.png -| | | | |- 📜 binarystepactivation.png -| | | | |- 📜 sigmoidactivation.png -| | | | |- 📜 neuronWithEquation.png +| | | | |- 📜 react-logo.png +| | | | |- 📜 pandas-logo.png | | | |- 📂 wiki_images: | | | | |- 📜 sigmoid_equation.png | | | | |- 📜 conv2d.gif -| | | | |- 📜 conv2d2.gif -| | | | |- 📜 avgpool_maxpool.gif | | | | |- 📜 softmax_equation.png : PNG file of Softmax equation -| | | | |- 📜 dropout_diagram.png | | | | |- 📜 batchnorm_diagram.png +| | | | |- 📜 dropout_diagram.png | | | | |- 📜 maxpool2d.gif +| | | | |- 📜 conv2d2.gif | | | | |- 📜 tanh_equation.png +| | | | |- 📜 avgpool_maxpool.gif | | | | |- 📜 tanh_plot.png | | | |- 📜 demo_video.gif : GIF tutorial of a simple classification training session -| | |- 📜 dlp-logo.ico : DLP Logo | | |- 📜 manifest.json : Default React file for choosing icon based on -| | |- 📜 index.html : Base HTML file that will be initially rendered | | |- 📜 robots.txt +| | |- 📜 index.html : Base HTML file that will be initially rendered +| | |- 📜 dlp-logo.ico : DLP Logo | |- 📂 layer_docs: | | |- 📜 Softmax.md : Doc for Softmax layer -| | |- 📜 Linear.md : Doc for Linear layer | | |- 📜 softmax_equation.png : PNG file of Softmax equation +| | |- 📜 Linear.md : Doc for Linear layer | | |- 📜 ReLU.md : Doc for ReLU later | |- 📂 src: -| | |- 📂 __tests__: -| | | |- 📂 common: -| | | | |- 📂 components: -| | | | | |- 📜 TitleText.test.tsx -| | |- 📂 backend_outputs: -| | | |- 📜 my_deep_learning_model.onnx : Last ONNX file output -| | | |- 📜 model.pkl -| | | |- 📜 model.pt : Last model.pt output -| | |- 📂 common: -| | | |- 📂 styles: -| | | | |- 📜 Home.module.css -| | | | |- 📜 globals.css -| | | |- 📂 redux: -| | | | |- 📜 hooks.ts -| | | | |- 📜 store.ts -| | | | |- 📜 train.ts -| | | | |- 📜 backendApi.ts -| | | | |- 📜 userLogin.ts -| | | |- 📂 utils: -| | | | |- 📜 dateFormat.ts -| | | | |- 📜 firebase.ts -| | | | |- 📜 dndHelpers.ts -| | | |- 📂 components: -| | | | |- 📜 EmailInput.tsx -| | | | |- 📜 HtmlTooltip.tsx -| | | | |- 📜 NavBarMain.tsx -| | | | |- 📜 Footer.tsx -| | | | |- 📜 DlpTooltip.tsx -| | | | |- 📜 ClientOnlyPortal.tsx -| | | | |- 📜 Spacer.tsx -| | | | |- 📜 TitleText.tsx +| | |- 📂 pages: +| | | |- 📂 train: +| | | | |- 📜 [train_space_id].tsx +| | | | |- 📜 index.tsx +| | | | |- 📜 metrics_to_charts.tsx +| | | |- 📜 settings.tsx +| | | |- 📜 dashboard.tsx +| | | |- 📜 login.tsx +| | | |- 📜 learn.tsx +| | | |- 📜 _app.tsx +| | | |- 📜 _document.tsx +| | | |- 📜 LearnContent.tsx +| | | |- 📜 feedback.tsx +| | | |- 📜 forgot.tsx +| | | |- 📜 wiki.tsx +| | | |- 📜 about.tsx | | |- 📂 features: -| | | |- 📂 LearnMod: -| | | | |- 📜 MCQuestion.tsx -| | | | |- 📜 ModulesSideBar.tsx -| | | | |- 📜 ImageComponent.tsx -| | | | |- 📜 ClassCard.tsx -| | | | |- 📜 FRQuestion.tsx -| | | | |- 📜 Exercise.tsx -| | | | |- 📜 LearningModulesContent.tsx -| | | |- 📂 OpenAi: -| | | | |- 📜 openAiUtils.ts -| | | |- 📂 Dashboard: -| | | | |- 📂 redux: -| | | | | |- 📜 dashboardApi.ts -| | | | |- 📂 components: -| | | | | |- 📜 TrainBarChart.tsx -| | | | | |- 📜 TrainDoughnutChart.tsx -| | | | | |- 📜 TrainDataGrid.tsx | | | |- 📂 Train: -| | | | |- 📂 redux: -| | | | | |- 📜 trainspaceSlice.ts -| | | | | |- 📜 trainspaceApi.ts -| | | | |- 📂 types: -| | | | | |- 📜 trainTypes.ts -| | | | |- 📂 constants: -| | | | | |- 📜 trainConstants.ts +| | | | |- 📂 components: +| | | | | |- 📜 DatasetStepLayout.tsx +| | | | | |- 📜 CreateTrainspace.tsx +| | | | | |- 📜 TrainspaceLayout.tsx | | | | |- 📂 features: -| | | | | |- 📂 Tabular: -| | | | | | |- 📂 redux: -| | | | | | | |- 📜 tabularActions.ts -| | | | | | | |- 📜 tabularApi.ts -| | | | | | |- 📂 types: -| | | | | | | |- 📜 tabularTypes.ts +| | | | | |- 📂 Image: +| | | | | | |- 📂 components: +| | | | | | | |- 📜 ImageFlow.tsx +| | | | | | | |- 📜 ImageParametersStep.tsx +| | | | | | | |- 📜 ImageDatasetStep.tsx +| | | | | | | |- 📜 ImageReviewStep.tsx +| | | | | | | |- 📜 ImageTrainspace.tsx | | | | | | |- 📂 constants: -| | | | | | | |- 📜 tabularConstants.ts +| | | | | | | |- 📜 imageConstants.ts +| | | | | | |- 📂 types: +| | | | | | | |- 📜 imageTypes.ts +| | | | | | |- 📂 redux: +| | | | | | | |- 📜 imageApi.ts +| | | | | | | |- 📜 imageActions.ts +| | | | | | |- 📜 index.ts +| | | | | |- 📂 Tabular: | | | | | | |- 📂 components: +| | | | | | | |- 📜 TabularParametersStep.tsx +| | | | | | | |- 📜 TabularFlow.tsx | | | | | | | |- 📜 TabularTrainspace.tsx | | | | | | | |- 📜 TabularReviewStep.tsx -| | | | | | | |- 📜 TabularParametersStep.tsx | | | | | | | |- 📜 TabularDatasetStep.tsx -| | | | | | | |- 📜 TabularFlow.tsx -| | | | | | |- 📜 index.ts -| | | | | |- 📂 Image: -| | | | | | |- 📂 redux: -| | | | | | | |- 📜 imageApi.ts -| | | | | | | |- 📜 imageActions.ts -| | | | | | |- 📂 types: -| | | | | | | |- 📜 imageTypes.ts | | | | | | |- 📂 constants: -| | | | | | | |- 📜 imageConstants.ts -| | | | | | |- 📂 components: -| | | | | | | |- 📜 ImageReviewStep.tsx -| | | | | | | |- 📜 ImageTrainspace.tsx -| | | | | | | |- 📜 ImageFlow.tsx -| | | | | | | |- 📜 ImageParametersStep.tsx -| | | | | | | |- 📜 ImageDatasetStep.tsx +| | | | | | | |- 📜 tabularConstants.ts +| | | | | | |- 📂 types: +| | | | | | | |- 📜 tabularTypes.ts +| | | | | | |- 📂 redux: +| | | | | | | |- 📜 tabularActions.ts +| | | | | | | |- 📜 tabularApi.ts | | | | | | |- 📜 index.ts -| | | | |- 📂 components: -| | | | | |- 📜 CreateTrainspace.tsx -| | | | | |- 📜 DatasetStepLayout.tsx -| | | | | |- 📜 TrainspaceLayout.tsx +| | | | |- 📂 constants: +| | | | | |- 📜 trainConstants.ts +| | | | |- 📂 types: +| | | | | |- 📜 trainTypes.ts +| | | | |- 📂 redux: +| | | | | |- 📜 trainspaceApi.ts +| | | | | |- 📜 trainspaceSlice.ts +| | | |- 📂 LearnMod: +| | | | |- 📜 FRQuestion.tsx +| | | | |- 📜 Exercise.tsx +| | | | |- 📜 ClassCard.tsx +| | | | |- 📜 ImageComponent.tsx +| | | | |- 📜 ModulesSideBar.tsx +| | | | |- 📜 MCQuestion.tsx +| | | | |- 📜 LearningModulesContent.tsx | | | |- 📂 Feedback: | | | | |- 📂 redux: | | | | | |- 📜 feedbackApi.ts -| | |- 📂 pages: -| | | |- 📂 train: -| | | | |- 📜 [train_space_id].tsx -| | | | |- 📜 metrics_to_charts.tsx -| | | | |- 📜 index.tsx -| | | |- 📜 _app.tsx -| | | |- 📜 forgot.tsx -| | | |- 📜 about.tsx -| | | |- 📜 settings.tsx -| | | |- 📜 _document.tsx -| | | |- 📜 feedback.tsx -| | | |- 📜 dashboard.tsx -| | | |- 📜 learn.tsx -| | | |- 📜 LearnContent.tsx -| | | |- 📜 login.tsx -| | | |- 📜 wiki.tsx -| | |- 📜 constants.ts +| | | |- 📂 OpenAi: +| | | | |- 📜 openAiUtils.ts +| | | |- 📂 Dashboard: +| | | | |- 📂 components: +| | | | | |- 📜 TrainDataGrid.tsx +| | | | | |- 📜 TrainBarChart.tsx +| | | | | |- 📜 TrainDoughnutChart.tsx +| | | | |- 📂 redux: +| | | | | |- 📜 dashboardApi.ts +| | |- 📂 common: +| | | |- 📂 components: +| | | | |- 📜 EmailInput.tsx +| | | | |- 📜 ClientOnlyPortal.tsx +| | | | |- 📜 Footer.tsx +| | | | |- 📜 TitleText.tsx +| | | | |- 📜 NavBarMain.tsx +| | | | |- 📜 Spacer.tsx +| | | | |- 📜 DlpTooltip.tsx +| | | | |- 📜 HtmlTooltip.tsx +| | | |- 📂 utils: +| | | | |- 📜 dateFormat.ts +| | | | |- 📜 firebase.ts +| | | | |- 📜 dndHelpers.ts +| | | |- 📂 styles: +| | | | |- 📜 globals.css +| | | | |- 📜 Home.module.css +| | | |- 📂 redux: +| | | | |- 📜 train.ts +| | | | |- 📜 backendApi.ts +| | | | |- 📜 userLogin.ts +| | | | |- 📜 hooks.ts +| | | | |- 📜 store.ts +| | |- 📂 __tests__: +| | | |- 📂 common: +| | | | |- 📂 components: +| | | | | |- 📜 TitleText.test.tsx +| | |- 📂 backend_outputs: +| | | |- 📜 my_deep_learning_model.onnx : Last ONNX file output +| | | |- 📜 model.pt : Last model.pt output +| | | |- 📜 model.pkl +| | |- 📜 next-env.d.ts | | |- 📜 iris.csv : Sample CSV data | | |- 📜 GlobalStyle.ts -| | |- 📜 next-env.d.ts -| |- 📜 pnpm-lock.yaml +| | |- 📜 constants.ts | |- 📜 tsconfig.json +| |- 📜 pnpm-lock.yaml | |- 📜 package.json -| |- 📜 .eslintrc.json | |- 📜 next.config.js -| |- 📜 next-env.d.ts | |- 📜 jest.config.ts +| |- 📜 next-env.d.ts | |- 📜 .eslintignore +| |- 📜 .eslintrc.json ``` diff --git a/.github/workflows/push-django-ecs.yml b/.github/workflows/push-django-ecs.yml index 08c3db64..59d2784d 100644 --- a/.github/workflows/push-django-ecs.yml +++ b/.github/workflows/push-django-ecs.yml @@ -62,7 +62,7 @@ jobs: - name: Login to Amazon ECR id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 + uses: aws-actions/amazon-ecr-login@v2 - name: Build, tag, and push image to Amazon ECR id: build-image @@ -73,13 +73,14 @@ jobs: # Build a docker container and # push it to ECR so that it can # be deployed to ECS. - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG training -f training/Dockerfile.prod + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG training -f training/Dockerfile --target production docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT - name: Download task definition run: | aws ecs describe-task-definition --task-definition django --query taskDefinition > temp-task-definition.json + - name: Fill in the new image ID in the Amazon ECS task definition id: task-def uses: aws-actions/amazon-ecs-render-task-definition@v1 @@ -89,7 +90,7 @@ jobs: image: ${{ steps.build-image.outputs.image }} - name: Deploy Amazon ECS task definition - uses: aws-actions/amazon-ecs-deploy-task-definition@v1 + uses: aws-actions/amazon-ecs-deploy-task-definition@v1.5.0 with: task-definition: ${{ steps.task-def.outputs.task-definition }} service: ${{ env.ECS_SERVICE }} diff --git a/.github/workflows/training-container.yml b/.github/workflows/push-training-ecs.yml similarity index 83% rename from .github/workflows/training-container.yml rename to .github/workflows/push-training-ecs.yml index dd536ec8..7c5d60f4 100644 --- a/.github/workflows/training-container.yml +++ b/.github/workflows/push-training-ecs.yml @@ -32,14 +32,14 @@ on: env: AWS_REGION: "us-east-1" # set this to your preferred AWS region, e.g. us-west-1 - ECR_REPOSITORY: "dlp-backend-image" # set this to your Amazon ECR repository name - ECS_SERVICE: "BackendService" # set this to your Amazon ECS service name - ECS_CLUSTER: "BackendCluster" # set this to your Amazon ECS cluster name - ECS_TASK_DEFINITION: - ".aws/training-task-definition.json" # set this to the path to your Amazon ECS task definition + ECR_REPOSITORY: "training" # set this to your Amazon ECR repository name + ECS_SERVICE: "training" # set this to your Amazon ECS service name + ECS_CLUSTER: "backend" # set this to your Amazon ECS cluster name + # ECS_TASK_DEFINITION: + # ".aws/training-task-definition.json" # set this to the path to your Amazon ECS task definition # file, e.g. .aws/task-definition.json CONTAINER_NAME: - "backend" # set this to the name of the container in the + "training" # set this to the name of the container in the # containerDefinitions section of your task definition permissions: @@ -67,7 +67,7 @@ jobs: - name: Login to Amazon ECR id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 + uses: aws-actions/amazon-ecr-login@v2 - name: Build, tag, and push image to Amazon ECR id: build-image @@ -78,20 +78,24 @@ jobs: # Build a docker container and # push it to ECR so that it can # be deployed to ECS. - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG training -f training/Dockerfile.prod + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG training -f training/training/core/celery/Dockerfile docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT + - name: Download task definition + run: | + aws ecs describe-task-definition --task-definition training --query taskDefinition > temp-task-definition.json + - name: Fill in the new image ID in the Amazon ECS task definition id: task-def uses: aws-actions/amazon-ecs-render-task-definition@v1 with: - task-definition: ${{ env.ECS_TASK_DEFINITION }} + task-definition: temp-task-definition.json container-name: ${{ env.CONTAINER_NAME }} image: ${{ steps.build-image.outputs.image }} - name: Deploy Amazon ECS task definition - uses: aws-actions/amazon-ecs-deploy-task-definition@v1 + uses: aws-actions/amazon-ecs-deploy-task-definition@v1.5.0 with: task-definition: ${{ steps.task-def.outputs.task-definition }} service: ${{ env.ECS_SERVICE }} diff --git a/dlp-terraform/ecs/ecs.tf b/dlp-terraform/ecs/ecs.tf index 89cdaafd..a0a11245 100644 --- a/dlp-terraform/ecs/ecs.tf +++ b/dlp-terraform/ecs/ecs.tf @@ -44,22 +44,7 @@ data "aws_iam_policy_document" "ecs_task_doc" { } } -resource "aws_iam_role" "ecs_task_role" { - name_prefix = "backend-ecs-task-role" - assume_role_policy = data.aws_iam_policy_document.ecs_task_doc.json -} - -resource "aws_iam_role_policy_attachment" "ecs_task_role_policy" { - for_each = toset([ - "arn:aws:iam::aws:policy/AmazonDynamoDBFullAccess", - "arn:aws:iam::aws:policy/SecretsManagerReadWrite" - ]) - - role = aws_iam_role.ecs_task_role.name - policy_arn = each.value -} - - +# --- ECS Exec Role --- resource "aws_iam_role" "ecs_exec_role" { name_prefix = "backend-ecs-exec-role" assume_role_policy = data.aws_iam_policy_document.ecs_task_doc.json diff --git a/dlp-terraform/ecs/ecs_django_service.tf b/dlp-terraform/ecs/ecs_django_service.tf index 6a2168d7..943e68c1 100644 --- a/dlp-terraform/ecs/ecs_django_service.tf +++ b/dlp-terraform/ecs/ecs_django_service.tf @@ -1,20 +1,43 @@ +# --- ECS Task Role --- # +resource "aws_iam_role" "django_ecs_task_role" { + name_prefix = "django-ecs-task-role" + assume_role_policy = data.aws_iam_policy_document.ecs_task_doc.json + + inline_policy { + name = "django-inline-policy" + policy = data.aws_iam_policy_document.django_inline_policy.json + } +} + +data "aws_iam_policy_document" "django_inline_policy" { + statement { + actions = ["sqs:SendMessage", "sqs:GetQueueAttributes"] + resources = [aws_sqs_queue.training_queue.arn] + } + + statement { + actions = ["secretsmanager:GetSecretValue"] + resources = ["arn:aws:secretsmanager:us-east-1:521654603461:secret:DLP/Firebase/Admin_SDK-8g8IDn"] + } +} + resource "aws_ecs_task_definition" "django" { - family = "django" - task_role_arn = aws_iam_role.ecs_task_role.arn - execution_role_arn = aws_iam_role.ecs_exec_role.arn - network_mode = "awsvpc" + family = "django" + task_role_arn = aws_iam_role.django_ecs_task_role.arn + execution_role_arn = aws_iam_role.ecs_exec_role.arn + network_mode = "awsvpc" requires_compatibilities = ["FARGATE"] - cpu = 1024 - memory = 2048 + cpu = 1024 + memory = 2048 container_definitions = jsonencode([ { - "name": "django", + "name" : "django", "image" : "${aws_ecr_repository.django.repository_url}:latest", - "cpu": 1024, - "memory": 2048, - "essential": true, - "portMappings": [ + "cpu" : 1024, + "memory" : 2048, + "essential" : true, + "portMappings" : [ { "name" : "gunicorn-port", "containerPort" : 8000, @@ -31,10 +54,10 @@ resource "aws_ecs_task_definition" "django" { "awslogs-stream-prefix" : "ecs" } }, - "environment": [ + "environment" : [ { - "name": "ALLOWED_HOST", - "value": "${aws_lb.main.dns_name}" + "name" : "ALLOWED_HOST", + "value" : "${aws_lb.main.dns_name}" } ] } @@ -50,39 +73,39 @@ resource "aws_security_group" "ecs_django_sg" { resource "aws_vpc_security_group_ingress_rule" "ecs_django_sg_ingress" { security_group_id = aws_security_group.ecs_django_sg.id - ip_protocol = "-1" + ip_protocol = "-1" referenced_security_group_id = aws_security_group.http.id -} +} resource "aws_vpc_security_group_egress_rule" "ecs_django_sg_egress" { security_group_id = aws_security_group.ecs_django_sg.id - ip_protocol = "-1" - cidr_ipv4 = "0.0.0.0/0" -} + ip_protocol = "-1" + cidr_ipv4 = "0.0.0.0/0" +} resource "aws_ecs_service" "django" { name = "django" cluster = aws_ecs_cluster.main.id task_definition = aws_ecs_task_definition.django.arn - desired_count = 2 - launch_type = "FARGATE" + desired_count = 1 + launch_type = "FARGATE" network_configuration { - security_groups = [ aws_security_group.ecs_django_sg.id] - subnets = aws_subnet.public[*].id + security_groups = [aws_security_group.ecs_django_sg.id] + subnets = aws_subnet.public[*].id assign_public_ip = true } lifecycle { - ignore_changes = [desired_count] + ignore_changes = [desired_count, task_definition] } load_balancer { target_group_arn = aws_lb_target_group.app.arn - container_name = "django" - container_port = 8000 + container_name = "django" + container_port = 8000 } depends_on = [aws_lb_target_group.app] -} \ No newline at end of file +} diff --git a/dlp-terraform/ecs/ecs_training_service.tf b/dlp-terraform/ecs/ecs_training_service.tf index 02ebb059..bd25a74a 100644 --- a/dlp-terraform/ecs/ecs_training_service.tf +++ b/dlp-terraform/ecs/ecs_training_service.tf @@ -1,6 +1,30 @@ +# --- ECS Task Role --- # +resource "aws_iam_role" "training_ecs_task_role" { + name_prefix = "training-ecs-task-role" + assume_role_policy = data.aws_iam_policy_document.ecs_task_doc.json + + inline_policy { + name = "training-inline-policy" + policy = data.aws_iam_policy_document.training_inline_policy.json + } +} + +data "aws_iam_policy_document" "training_inline_policy" { + statement { + actions = ["sqs:ReceiveMessage", "sqs:GetQueueAttributes", "sqs:ChangeMessageVisibility", "sqs:DeleteMessage"] + resources = [aws_sqs_queue.training_queue.arn] + } + + statement { + actions = ["s3:GetObject", "s3:PutObject"] + resources = [aws_s3_bucket.s3bucket_executions.arn, "${aws_s3_bucket.s3bucket_executions.arn}/*"] + } +} + +# --- ECS Task Definition --- # resource "aws_ecs_task_definition" "training" { family = "training" - task_role_arn = aws_iam_role.ecs_task_role.arn + task_role_arn = aws_iam_role.training_ecs_task_role.arn execution_role_arn = aws_iam_role.ecs_exec_role.arn network_mode = "bridge" cpu = 1024 @@ -10,15 +34,6 @@ resource "aws_ecs_task_definition" "training" { { "name" : "training", "image" : "${aws_ecr_repository.training.repository_url}:latest", - "portMappings" : [ - { - "name" : "gunicorn-port", - "containerPort" : 8000, - "hostPort" : 0, - "protocol" : "tcp", - "appProtocol" : "http" - } - ], "essential" : true, "environment" : [], "mountPoints" : [], @@ -55,32 +70,24 @@ resource "aws_ecs_service" "training" { } lifecycle { - ignore_changes = [desired_count] + ignore_changes = [desired_count, task_definition] } - - # load_balancer { - # target_group_arn = aws_lb_target_group.app.arn - # container_name = "training" - # container_port = 8000 - # } - - # depends_on = [aws_lb_target_group.app] } # --- ECS Service Auto Scaling --- resource "aws_appautoscaling_target" "training_ecs_target" { - service_namespace = "ecs" + service_namespace = "ecs" scalable_dimension = "ecs:service:DesiredCount" - resource_id = "service/${aws_ecs_cluster.main.name}/${aws_ecs_service.training.name}" - min_capacity = 0 - max_capacity = 2 + resource_id = "service/${aws_ecs_cluster.main.name}/${aws_ecs_service.training.name}" + min_capacity = 0 + max_capacity = 2 } resource "aws_appautoscaling_policy" "training_ecs_target_cpu" { - name = "training-application-scaling-policy-cpu" - policy_type = "TargetTrackingScaling" - service_namespace = aws_appautoscaling_target.training_ecs_target.service_namespace - resource_id = aws_appautoscaling_target.training_ecs_target.resource_id + name = "training-application-scaling-policy-cpu" + policy_type = "TargetTrackingScaling" + service_namespace = aws_appautoscaling_target.training_ecs_target.service_namespace + resource_id = aws_appautoscaling_target.training_ecs_target.resource_id scalable_dimension = aws_appautoscaling_target.training_ecs_target.scalable_dimension target_tracking_scaling_policy_configuration { @@ -88,17 +95,17 @@ resource "aws_appautoscaling_policy" "training_ecs_target_cpu" { predefined_metric_type = "ECSServiceAverageCPUUtilization" } - target_value = 80 - scale_in_cooldown = 300 + target_value = 80 + scale_in_cooldown = 300 scale_out_cooldown = 300 } } resource "aws_appautoscaling_policy" "training_ecs_target_memory" { - name = "training-application-scaling-policy-memory" - policy_type = "TargetTrackingScaling" - service_namespace = aws_appautoscaling_target.training_ecs_target.service_namespace - resource_id = aws_appautoscaling_target.training_ecs_target.resource_id + name = "training-application-scaling-policy-memory" + policy_type = "TargetTrackingScaling" + service_namespace = aws_appautoscaling_target.training_ecs_target.service_namespace + resource_id = aws_appautoscaling_target.training_ecs_target.resource_id scalable_dimension = aws_appautoscaling_target.training_ecs_target.scalable_dimension target_tracking_scaling_policy_configuration { @@ -106,8 +113,8 @@ resource "aws_appautoscaling_policy" "training_ecs_target_memory" { predefined_metric_type = "ECSServiceAverageMemoryUtilization" } - target_value = 80 - scale_in_cooldown = 300 + target_value = 80 + scale_in_cooldown = 300 scale_out_cooldown = 300 } } @@ -121,17 +128,17 @@ resource "aws_security_group" "ecs_training_sg" { resource "aws_vpc_security_group_ingress_rule" "ecs_training_sg_ingress" { security_group_id = aws_security_group.ecs_training_sg.id - ip_protocol = "-1" + ip_protocol = "-1" # cidr_blocks = [aws_vpc.main.cidr_block] referenced_security_group_id = aws_security_group.ecs_django_sg.id -} +} resource "aws_vpc_security_group_egress_rule" "ecs_training_sg_egress" { security_group_id = aws_security_group.ecs_training_sg.id - ip_protocol = "-1" - cidr_ipv4 = "0.0.0.0/0" -} + ip_protocol = "-1" + cidr_ipv4 = "0.0.0.0/0" +} # --- ECS Launch Template --- resource "aws_launch_template" "ecs_lt_training" { @@ -209,4 +216,4 @@ resource "aws_ecs_cluster_capacity_providers" "main" { base = 1 weight = 100 } -} \ No newline at end of file +} diff --git a/dlp-terraform/ecs/main.tf b/dlp-terraform/ecs/main.tf index f4506a15..d81f6d3b 100644 --- a/dlp-terraform/ecs/main.tf +++ b/dlp-terraform/ecs/main.tf @@ -9,7 +9,7 @@ terraform { provider "aws" { region = "us-east-1" - profile = "DLP_Deploy-521654603461" + profile = "dlp" } # --- VPC -- diff --git a/dlp-terraform/ecs/sqs.tf b/dlp-terraform/ecs/sqs.tf index d5ca2017..35186360 100644 --- a/dlp-terraform/ecs/sqs.tf +++ b/dlp-terraform/ecs/sqs.tf @@ -10,7 +10,8 @@ resource "aws_sqs_queue" "training_queue" { } resource "aws_sqs_queue" "training_queue_deadletter" { - name = "training-deadletter-queue" + name = "training-deadletter-queue.fifo" + fifo_queue = true } resource "aws_sqs_queue_redrive_allow_policy" "training_queue_redrive_allow_policy" { diff --git a/frontend/next.config.js b/frontend/next.config.js index 180c1257..51d2a54a 100644 --- a/frontend/next.config.js +++ b/frontend/next.config.js @@ -23,7 +23,10 @@ const nextConfig = { }, { source: "/api/training/:path*", - destination: "http://127.0.0.1:8000/api/:path*", + destination: + process.env.ENVIRONMENT === "production" + ? "http://alb-1805434018.us-east-1.elb.amazonaws.com/api/:path*" // note, this url changes every time you destroy/apply Terraform + : "http://127.0.0.1:8000/api/:path*", }, ], };