serverless.yml

service: data-portal-api

frameworkVersion: "^3"

# FIXME: https://github.com/serverless/serverless/pull/12300
#  see https://github.com/umccr/data-portal-apis/issues/718
#  allowed values [error, warn, off] turning it `warn` for python3.12 runtime
configValidationMode: warn

plugins:
  - serverless-wsgi
  - serverless-python-requirements
  - serverless-domain-manager
  #- serverless-associate-waf
  - serverless-prune-plugin

provider:
  name: aws
  deploymentMethod: direct
  runtime: python3.12
  architecture: arm64
  memorySize: 512
  stage: ${opt:stage, 'dev'}
  region: ap-southeast-2
  deploymentBucket:
    name: ${ssm:/data_portal/backend/serverless_deployment_bucket}
    serverSideEncryption: AES256
  iam:
    role: ${ssm:/data_portal/backend/lambda_iam_role_arn}
  endpointType: REGIONAL
  logs:
    httpApi: true
  httpApi:
    disableDefaultEndpoint: true
    payload: "2.0"
    cors:
      allowedOrigins:
        - https://portal.umccr.org
        - https://portal.prod.umccr.org
        - https://portal.dev.umccr.org
        - https://sscheck.umccr.org
        - https://sscheck.prod.umccr.org
        - https://sscheck.dev.umccr.org
      allowedHeaders:
        - Content-Type
        - Content-Disposition
        - Authorization
        - X-Amz-Date
        - X-Api-Key
        - X-Amz-Security-Token
        - X-Amz-User-Agent
      allowedMethods:
        - GET
        - OPTIONS
        - HEAD
        - DELETE
        - POST
        - PUT
        - PATCH
      allowCredentials: false
      maxAge: 300
    authorizers:
      cognitoJwtAuthorizer:
        type: jwt
        identitySource: $request.header.Authorization
        issuerUrl: https://cognito-idp.${self:provider.region}.amazonaws.com/${ssm:/data_portal/client/cog_user_pool_id}
        audience:
          - ${ssm:/data_portal/client/cog_app_client_id_local}
          - ${ssm:/data_portal/client/cog_app_client_id_stage}
          - ${ssm:/data_portal/client/data2/cog_app_client_id_stage}
          - ${ssm:/sscheck/client/cog_app_client_id_stage}
  vpc:
    securityGroupIds:
      "Fn::Split":
        - ","
        - ${ssm:/data_portal/backend/lambda_security_group_ids}
    subnetIds:
      "Fn::Split":
        - ","
        - ${ssm:/data_portal/backend/lambda_subnet_ids}
  environment:
    DJANGO_SETTINGS_MODULE: data_portal.settings.aws
    LAB_METADATA_SYNC_LAMBDA: ${self:service}-${sls:stage}-labmetadata_scheduled_update_processor

functions:
  api:
    handler: wsgi_handler.handler
    snapStart: true
    # configure API lambda to use rds aurora reader endpoint
    environment:
      DJANGO_SETTINGS_MODULE: data_portal.settings.aws_ro
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    events:
      - httpApi:
          path: /{proxy+}
          method: GET
          authorizer: cognitoJwtAuthorizer
      - httpApi:
          path: /{proxy+}
          method: POST
          authorizer: cognitoJwtAuthorizer
      # we mirror the API onto /iam/* and allow IAM access
      - httpApi:
          path: /iam/{proxy+}
          method: GET
          authorizer:
            type: aws_iam
      - httpApi:
          path: /iam/{proxy+}
          method: POST
          authorizer:
            type: aws_iam
    timeout: 28

  migrate:
    handler: migrate.handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 120

  lims_scheduled_update_processor:
    handler: data_processors.lims.lambdas.google_lims.scheduled_update_handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    events:
      - schedule:
          rate: cron(0 12 * * ? *)
          enabled: ${self:custom.enabled.${self:provider.stage}, self:custom.enabled.other}
    timeout: 360

  labmetadata_scheduled_update_processor:
    handler: data_processors.lims.lambdas.labmetadata.scheduled_update_handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    events:
      - schedule:
          rate: cron(0 12 * * ? *)
          enabled: ${self:custom.enabled.${self:provider.stage}, self:custom.enabled.other}
    timeout: 360

  sqs_s3_event_processor:
    handler: data_processors.s3.lambdas.s3_event.handler
    memorySize: 256
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    events:
      - sqs:
          arn: ${ssm:/data_portal/backend/s3_event_sqs_arn}
    timeout: 120
    reservedConcurrency: 20
    # Uncomment to enable xray on this lambda
  #    tracing: Active
  #    environment:
  #      AWS_XRAY_SDK_ENABLED: true

  sqs_gds_event_processor:
    handler: data_processors.gds.lambdas.gds_event.handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    events:
      - sqs:
          arn: ${ssm:/data_portal/backend/ica_gds_event_sqs_arn}
    timeout: 120
    reservedConcurrency: 20

  sqs_batch_event_processor:
    handler: data_processors.pipeline.lambdas.sqs_batch_event.handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    events:
      - sqs:
          arn: ${ssm:/data_portal/backend/batch_event_sqs_arn}
    timeout: 120

  sqs_iap_event_processor:
    handler: data_processors.pipeline.lambdas.sqs_iap_event.handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    events:
      - sqs:
          arn: ${ssm:/data_portal/backend/iap_ens_event_sqs_arn}
    timeout: 120

  # SQS Implement partial batch responses - ReportBatchItemFailures
  # https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#services-sqs-batchfailurereporting
  # https://repost.aws/knowledge-center/lambda-sqs-report-batch-item-failures
  sqs_dragen_wgs_qc_event_processor:
    handler: data_processors.pipeline.lambdas.dragen_wgs_qc.sqs_handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    events:
      - sqs:
          arn: ${ssm:/data_portal/backend/sqs_dragen_wgs_qc_queue_arn}
          functionResponseType: ReportBatchItemFailures
    timeout: 28

  sqs_dragen_wts_event_processor:
    handler: data_processors.pipeline.lambdas.dragen_wts.sqs_handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    events:
      - sqs:
          arn: ${ssm:/data_portal/backend/sqs_dragen_wts_queue_arn}
          functionResponseType: ReportBatchItemFailures
    timeout: 28

  sqs_star_alignment_event_processor:
    handler: data_processors.pipeline.lambdas.star_alignment.sqs_handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    events:
      - sqs:
          arn: ${ssm:/data_portal/backend/sqs_star_alignment_queue_arn}
          functionResponseType: ReportBatchItemFailures
    timeout: 28

  sqs_oncoanalyser_wts_event_processor:
    handler: data_processors.pipeline.lambdas.oncoanalyser_wts.sqs_handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    events:
      - sqs:
          arn: ${ssm:/data_portal/backend/sqs_oncoanalyser_wts_queue_arn}
          functionResponseType: ReportBatchItemFailures
    timeout: 28

  sqs_oncoanalyser_wgs_event_processor:
    handler: data_processors.pipeline.lambdas.oncoanalyser_wgs.sqs_handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    events:
      - sqs:
          arn: ${ssm:/data_portal/backend/sqs_oncoanalyser_wgs_queue_arn}
          functionResponseType: ReportBatchItemFailures
    timeout: 28

  sqs_oncoanalyser_wgts_existing_both:
    handler: data_processors.pipeline.lambdas.oncoanalyser_wgts_existing_both.sqs_handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    events:
      - sqs:
          arn: ${ssm:/data_portal/backend/sqs_oncoanalyser_wgts_queue_arn}
          functionResponseType: ReportBatchItemFailures
    timeout: 28

  sqs_sash_event_processor:
    handler: data_processors.pipeline.lambdas.sash.sqs_handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    events:
      - sqs:
          arn: ${ssm:/data_portal/backend/sqs_sash_queue_arn}
          functionResponseType: ReportBatchItemFailures
    timeout: 28

  sqs_dragen_tso_ctdna_event_processor:
    handler: data_processors.pipeline.lambdas.dragen_tso_ctdna.sqs_handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    events:
      - sqs:
          arn: ${ssm:/data_portal/backend/sqs_dragen_tso_ctdna_queue_arn}
          functionResponseType: ReportBatchItemFailures
    timeout: 28

  sqs_tumor_normal_event_processor:
    handler: data_processors.pipeline.lambdas.tumor_normal.sqs_handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    events:
      - sqs:
          arn: ${ssm:/data_portal/backend/sqs_tumor_normal_queue_arn}
          functionResponseType: ReportBatchItemFailures
    timeout: 28

  sqs_umccrise_event_processor:
    handler: data_processors.pipeline.lambdas.umccrise.sqs_handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    events:
      - sqs:
          arn: ${ssm:/data_portal/backend/sqs_umccrise_queue_arn}
          functionResponseType: ReportBatchItemFailures
    timeout: 28

  sqs_rnasum_event_processor:
    handler: data_processors.pipeline.lambdas.rnasum.sqs_handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    events:
      - sqs:
          arn: ${ssm:/data_portal/backend/sqs_rnasum_queue_arn}
          functionResponseType: ReportBatchItemFailures
    timeout: 28

  sqs_somalier_extract_event_processor:
    handler: data_processors.pipeline.lambdas.somalier_extract.sqs_handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    events:
      - sqs:
          arn: ${ssm:/data_portal/backend/sqs_somalier_extract_queue_arn}
          functionResponseType: ReportBatchItemFailures
    timeout: 60

  sqs_notification_event_processor:
    handler: data_processors.pipeline.lambdas.notification.sqs_handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    events:
      - sqs:
          arn: ${ssm:/data_portal/backend/sqs_notification_queue_arn}
          functionResponseType: ReportBatchItemFailures
    timeout: 28

  bcl_convert:
    handler: data_processors.pipeline.lambdas.bcl_convert.handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 28

  dragen_wgs_qc:
    handler: data_processors.pipeline.lambdas.dragen_wgs_qc.handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 28

  dragen_wts:
    handler: data_processors.pipeline.lambdas.dragen_wts.handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 28

  star_alignment:
    handler: data_processors.pipeline.lambdas.star_alignment.handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 28

  oncoanalyser_wts:
    handler: data_processors.pipeline.lambdas.oncoanalyser_wts.handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 28

  oncoanalyser_wgs:
    handler: data_processors.pipeline.lambdas.oncoanalyser_wgs.handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 28

  oncoanalyser_wgts_existing_both:
    handler: data_processors.pipeline.lambdas.oncoanalyser_wgts_existing_both.handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 28

  sash:
    handler: data_processors.pipeline.lambdas.sash.handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 28

  dragen_tso_ctdna:
    handler: data_processors.pipeline.lambdas.dragen_tso_ctdna.handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 28

  tumor_normal:
    handler: data_processors.pipeline.lambdas.tumor_normal.handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 28

  umccrise:
    handler: data_processors.pipeline.lambdas.umccrise.handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 28

  rnasum:
    handler: data_processors.pipeline.lambdas.rnasum.handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 28

  rnasum_by_umccrise:
    handler: data_processors.pipeline.lambdas.rnasum.by_umccrise_handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 28

  rnasum_by_subject:
    handler: data_processors.pipeline.lambdas.rnasum.by_subject_handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 28

  fastq:
    handler: data_processors.pipeline.lambdas.fastq.handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 28

  fastq_list_row:
    handler: data_processors.pipeline.lambdas.fastq_list_row.handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 28

  orchestrator:
    handler: data_processors.pipeline.lambdas.orchestrator.handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 28

  workflow_update:
    handler: data_processors.pipeline.lambdas.workflow_update.handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 28

  orchestrator_ng:
    handler: data_processors.pipeline.lambdas.orchestrator.handler_ng
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 28

  workflow_update_ng:
    handler: data_processors.pipeline.lambdas.workflow_update.handler_ng
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 28

  google_lims_update:
    handler: data_processors.pipeline.lambdas.google_lims_update.handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 28

  google_lims_update_by_provided_id:
    handler: data_processors.pipeline.lambdas.google_lims_update.by_provided_id
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 120

  wes_launch:
    handler: data_processors.pipeline.lambdas.wes_handler.launch
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 28

  wes_get_workflow_run:
    handler: data_processors.pipeline.lambdas.wes_handler.get_workflow_run
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 28

  notification:
    handler: data_processors.pipeline.lambdas.notification.handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 28

  gds_search:
    handler: data_processors.pipeline.lambdas.gds_search.handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 28

  libraryrun:
    handler: data_processors.pipeline.lambdas.libraryrun.handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 60

  somalier_extract:
    handler: data_processors.pipeline.lambdas.somalier_extract.handler
    layers:
      - { Ref: PythonRequirementsLambdaLayer }
    timeout: 60

custom:
  enabled:
    dev: false
    prod: true
    stg: true
    other: false
  wsgi:
    app: data_portal.wsgi.application
    packRequirements: false
  pythonRequirements:
    #
    # See https://github.com/serverless/serverless-python-requirements/pull/724
    # It should auto discover and use appropriate SAM build image by default.
    # If this is not working on your local OS/platform, adjust the `dockerImage`
    # to try with difference build image. Please note that the actual CodeBuild
    # environment does not require this. Only for local `sls deploy --stage dev`
    # on macOS. e.g. direct push to DEV for debugging build convenience.
    #
    #dockerImage: public.ecr.aws/sam/build-python3.12
    dockerizePip: non-linux
    layer: true
    zip: false
    slim: true
    slimPatternsAppendDefaults: true
    slimPatterns: ${file(slimpatterns.yml):slimPatterns}
  customDomains:
    - http:
        domainName: ${ssm:/data_portal/backend/api_domain_name2}
        basePath: ""
        # Either set the stage to proper one or use the APIGateway v2 $default value
        #        stage: ${self:provider.stage}
        createRoute53Record: true
        certificateArn: ${ssm:/data_portal/backend/certificate_arn2}
        apiType: http
        endpointType: regional
  #associateWaf:
  #  name: ${ssm:/data_portal/backend/waf_name}
  prune:
    automatic: true
    includeLayers: true
    number: 3

package:
  # See https://www.serverless.com/framework/docs/providers/aws/guide/packaging/
  patterns:
    - "!.husky/**"
    - "!.git/**"
    - "!.idea/**"
    - "!__pycache__/**"
    - "!node_modules/**"
    - "!package.json"
    - "!yarn.lock"
    - "!mocks/**"
    - "!downloads/**"
    - "!venv/**"
    - "!.venv/**"
    - "!env/**"
    - "!.env/**"
    - "!data/**"
    - "!swagger/**"
    - "!docs/**"
    - "!docker-compose.yml"
    - "!docker-compose.override.sample.yml"
    - "!docker-compose.override.yml"
    - "!docker-compose.ci.yml"
    - "!buildspec.yml"
    - "!slimpatterns.yml"
    - "!Makefile"
    - "!README.md"
    - "!requirements-dev.txt"
    - "!requirements-test.txt"
    - "!haproxy.cfg"
    - "!loaddata.sh"