feat(workflows-api): add workflow api and supporting deployment script

infrastructure/cf_update.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+CLOUDFRONT_DIST_ID=$1
+WORKFLOWS_API_ORIGIN_ID=$2
+WORKFLOWS_API_DOMAIN_NAME=$3
+
+WORKFLOWS_API_DOMAIN_NAME=${WORKFLOWS_API_DOMAIN_NAME#http://}
+WORKFLOWS_API_DOMAIN_NAME=${WORKFLOWS_API_DOMAIN_NAME#https://}
+
+# Step 1: Retrieve the current CloudFront distribution configuration
+aws cloudfront get-distribution-config --id $CLOUDFRONT_DIST_ID > current-config.json
+
+# Extract the ETag and DistributionConfig for later use
+ETAG=$(jq -r '.ETag' current-config.json)
+DISTRIBUTION_CONFIG=$(jq '.DistributionConfig' current-config.json)
+
+# Step 2: Modify the configuration
+# Add a new origin for the workflows API
+MODIFIED_CONFIG=$(echo $DISTRIBUTION_CONFIG | jq --arg origin_id "$WORKFLOWS_API_ORIGIN_ID" --arg domain_name "$WORKFLOWS_API_DOMAIN_NAME" '.Origins.Items += [{"Id": $origin_id, "DomainName": $domain_name, "CustomOriginConfig": {"HTTPPort": 80, "HTTPSPort": 443, "OriginProtocolPolicy": "https-only", "OriginSslProtocols": {"Items": ["TLSv1.2"], "Quantity": 1}, "OriginReadTimeout": 30, "OriginKeepaliveTimeout": 5}, "OriginPath": "", "CustomHeaders": {"Quantity": 0}}] | .Origins.Quantity += 1')
+# Add a new cache behavior for the workflows API
+MODIFIED_CONFIG=$(echo $MODIFIED_CONFIG | jq --arg origin_id "$WORKFLOWS_API_ORIGIN_ID" '.CacheBehaviors.Items += [{"PathPattern": "/api/workflows/*", "TargetOriginId": $origin_id, "ViewerProtocolPolicy": "redirect-to-https", "AllowedMethods": {"Items": ["GET", "HEAD", "OPTIONS", "PUT", "POST", "PATCH", "DELETE"], "Quantity": 7, "CachedMethods": {"Items": ["GET", "HEAD"], "Quantity": 2}}, "SmoothStreaming": false, "CachePolicyId": "4135ea2d-6df8-44a3-9df3-4b5a84be39ad", "LambdaFunctionAssociations": {"Quantity": 0},"FunctionAssociations": {"Quantity": 0}, "FieldLevelEncryptionId": "", "Compress": true}] | .CacheBehaviors.Quantity += 1')
+
+# Step 3: Update the CloudFront distribution with the modified configuration
+echo $MODIFIED_CONFIG | jq . > modified-config.json
+aws cloudfront update-distribution --id $CLOUDFRONT_DIST_ID --if-match $ETAG --distribution-config file://modified-config.json
+
+# Cleanup
+rm current-config.json modified-config.json

infrastructure/main.tf

@@ -105,3 +105,150 @@ resource "local_file" "mwaa_variables" {
   })
   filename = "/tmp/mwaa_vars.json"
 }
+
+##########################################################
+# Workflows API
+##########################################################
+
+# ECR repository to host workflows API image
+resource "aws_ecr_repository" "workflows_api_lambda_repository" {
+  name = "veda_${var.stage}_workflows-api-lambda-repository"
+} 
+
+resource "null_resource" "if_change_run_provisioner" {
+  triggers = {
+      always_run = "${timestamp()}"
+      }
+  provisioner "local-exec" {
+    command = <<-EOT
+      set -e
+      aws ecr get-login-password --region ${local.aws_region} | docker login --username AWS --password-stdin ${aws_ecr_repository.workflows_api_lambda_repository.repository_url}
+      docker build --platform=linux/amd64 -t ${aws_ecr_repository.workflows_api_lambda_repository.repository_url}:latest ../workflows_api/runtime/
+      docker push ${aws_ecr_repository.workflows_api_lambda_repository.repository_url}:latest
+    EOT
+  }
+}
+
+# IAM Role for Lambda Execution
+resource "aws_iam_role" "lambda_execution_role" {
+  name = "veda_${var.stage}_lambda_execution_role"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+        Action = "sts:AssumeRole",
+        Effect = "Allow",
+        Principal = {
+          Service = "lambda.amazonaws.com",
+        },
+      },
+    ],
+  })
+}
+
+resource "aws_iam_policy" "ecr_access" {
+  name        = "veda_${var.stage}_ECR_Access_For_Lambda"
+  path        = "/"
+  description = "ECR access policy for Lambda function"
+  policy      = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+        Effect   = "Allow",
+        Action   = [
+          "ecr:GetDownloadUrlForLayer",
+          "ecr:BatchGetImage",
+        ],
+        Resource = "${aws_ecr_repository.workflows_api_lambda_repository.arn}",
+      },
+    ],
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "ecr_access_attach" {
+  role       = aws_iam_role.lambda_execution_role.name
+  policy_arn = aws_iam_policy.ecr_access.arn
+}
+
+resource "aws_iam_role_policy_attachment" "lambda_basic_execution" {
+  role       = aws_iam_role.lambda_execution_role.name
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
+}
+
+
+resource "aws_lambda_function" "workflows_api_handler" {
+  function_name = "veda_${var.stage}_workflows_api_handler"
+  role          = aws_iam_role.lambda_execution_role.arn
+  package_type = "Image"
+  timeout = 30
+
+  image_uri = "${aws_ecr_repository.workflows_api_lambda_repository.repository_url}:latest"
+  environment {
+    variables = {
+      JWKS_URL          = var.jwks_url
+      USERPOOL_ID       = var.cognito_userpool_id
+      CLIENT_ID         = var.cognito_client_id
+      CLIENT_SECRET     = var.cognito_client_secret
+      STAGE             = var.stage
+      DATA_ACCESS_ROLE_ARN = var.data_access_role_arn
+      WORKFLOW_ROOT_PATH = var.workflow_root_path
+      INGEST_URL = var.ingest_url
+      RASTER_URL = var.raster_url
+      STAC_URL = var.stac_url
+      MWAA_ENV = module.mwaa.airflow_url
+    }
+  }
+}
+
+
+# API Gateway HTTP API
+resource "aws_apigatewayv2_api" "workflows_http_api" {
+  name          = "veda_${var.stage}_workflows_http_api"
+  protocol_type = "HTTP"
+}
+
+# Lambda Integration for API Gateway
+resource "aws_apigatewayv2_integration" "workflows_lambda_integration" {
+  api_id           = aws_apigatewayv2_api.workflows_http_api.id
+  integration_type = "AWS_PROXY"
+  integration_uri  = aws_lambda_function.workflows_api_handler.invoke_arn
+  payload_format_version = "2.0"
+}
+
+# Default Route for API Gateway
+resource "aws_apigatewayv2_route" "workflows_default_route" {
+  api_id    = aws_apigatewayv2_api.workflows_http_api.id
+  route_key = "$default"
+  target    = "integrations/${aws_apigatewayv2_integration.workflows_lambda_integration.id}"
+}
+
+resource "aws_apigatewayv2_stage" "workflows_default_stage" {
+  api_id      = aws_apigatewayv2_api.workflows_http_api.id
+  name        = "$default"
+  auto_deploy = true
+}
+
+resource "aws_lambda_permission" "api-gateway" {
+  statement_id  = "AllowExecutionFromAPIGateway"
+  action        = "lambda:InvokeFunction"
+  function_name = aws_lambda_function.workflows_api_handler.arn
+  principal     = "apigateway.amazonaws.com"
+  source_arn    = "${aws_apigatewayv2_api.workflows_http_api.execution_arn}/*/$default"
+}
+
+# Cloudfront update
+
+resource "null_resource" "update_cloudfront" {
+  triggers = {
+    always_run = "${timestamp()}"
+  }
+
+  count = var.cloudfront_id ? 1 : 0
+
+  provisioner "local-exec" {
+    command = "${path.module}/cf_update.sh ${var.cloudfront_id} workflows_api_origin \"${aws_apigatewayv2_api.workflows_http_api.api_endpoint}\""
+  }
+
+  depends_on = [aws_apigatewayv2_api.workflows_http_api]
+}

infrastructure/terraform.tfvars.tmpl

@@ -11,3 +11,12 @@ stac_ingestor_api_url="${STAC_INGESTOR_API_URL}"
 vector_secret_name="${VECTOR_SECRET_NAME}"
 vector_security_group="${VECTOR_SECURITY_GROUP}"
 vector_vpc="${VECTOR_VPC}"
+data_access_role_arn="${DATA_ACCESS_ROLE_ARN}"
+workflow_root_path="${WORKFLOW_ROOT_PATH}"
+ingest_url="${INGEST_URL}"
+raster_url="${RASTER_URL}"
+stac_url="${STAC_URL}"
+cloudfront_id="${CLOUDFRONT_ID}"
+jwks_url="${JWKS_URL}"
+cognito_userpool_id="${COGNITO_USERPOOL_ID}"
+cognito_client_id="${COGNITO_CLIENT_ID}"

infrastructure/variables.tf

@@ -37,6 +37,24 @@ variable "cognito_app_secret" {
   type = string
 }
 
+variable "jwks_url" {
+  type = string
+}
+
+variable "cognito_userpool_id" {
+  type = string
+}
+
+variable "cognito_client_id" {
+  type = string
+}
+
+variable "cognito_client_secret" {
+  description = "Optional secret, if required by cognito user pool"
+  default = ""
+  type = string
+}
+
 variable "stac_ingestor_api_url" {
   type = string
 }
@@ -58,3 +76,28 @@ variable "vector_vpc" {
   type = string
 }
 
+variable "data_access_role_arn" {
+  type = string
+}
+
+variable "workflow_root_path" {
+  type = string
+  default = "/api/workflows"
+}
+
+variable "ingest_url" {
+  type = string
+}
+
+variable "raster_url" {
+  type = string
+}
+
+variable "stac_url" {
+  type = string
+}
+
+variable "cloudfront_id" {
+  type = string
+}
+

workflows_api/runtime/Dockerfile

@@ -0,0 +1,8 @@
+FROM public.ecr.aws/lambda/python:3.9
+
+COPY ./requirements.txt ${LAMBDA_TASK_ROOT}
+RUN pip install -r ${LAMBDA_TASK_ROOT}/requirements.txt
+
+COPY . ${LAMBDA_TASK_ROOT}
+
+CMD [ "lambda_function.handler" ]

workflows_api/runtime/lambda_function.py

@@ -0,0 +1,10 @@
+"""
+Entrypoint for Lambda execution.
+"""
+
+from mangum import Mangum
+from src.main import workflows_app
+
+handler = Mangum(
+    workflows_app, lifespan="off", api_gateway_base_path=workflows_app.root_path
+)

workflows_api/runtime/local.py

@@ -0,0 +1,7 @@
+""" Script used to run the API locally"""
+
+import uvicorn
+from src import main
+
+if __name__ == "__main__":
+    uvicorn.run(main.workflows_app, host="0.0.0.0", port=8000)

workflows_api/runtime/requirements.txt

@@ -0,0 +1,19 @@
+# Waiting for https://github.com/stac-utils/stac-pydantic/pull/116 and 117
+Authlib==1.0.1
+ddbcereal==2.1.1
+fastapi>=0.75.1
+fsspec==2023.3.0
+mangum>=0.15.0
+orjson>=3.6.8
+psycopg[binary,pool]>=3.0.15
+pydantic_ssm_settings>=0.2.0
+pydantic>=1.9.0
+pypgstac==0.7.4
+python-multipart==0.0.5
+requests>=2.27.1
+s3fs==2023.3.0
+stac-pydantic
+xarray==2023.1.0
+xstac==1.1.0
+zarr==2.13.6
+boto3==1.24.59