-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
2 changed files
with
39 additions
and
25 deletions.
There are no files selected for viewing
32 changes: 7 additions & 25 deletions
32
...mbedder/relation_embedder/src/main/scala/weco/pipeline/relation_embedder/LambdaMain.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,39 +1,21 @@ | ||
package weco.pipeline.relation_embedder | ||
|
||
import com.amazonaws.services.lambda.runtime.{Context, RequestHandler} | ||
import grizzled.slf4j.Logging | ||
import com.amazonaws.services.lambda.runtime.events.SQSEvent | ||
import org.apache.pekko.actor.ActorSystem | ||
import weco.pipeline.relation_embedder.lib._ | ||
|
||
import scala.concurrent.duration.DurationInt | ||
import scala.concurrent.{Await, ExecutionContext, Future} | ||
import scala.concurrent.Future | ||
|
||
|
||
object LambdaMain | ||
extends RequestHandler[SQSEvent, String] | ||
with Logging | ||
with RelationEmbedderConfigurable { | ||
extends LambdaApp[SQSEvent, String, RelationEmbedderConfig] | ||
with RelationEmbedderConfigurable { | ||
|
||
import SQSEventOps._ | ||
private lazy val batchProcessor = BatchProcessor(config) | ||
|
||
override def handleRequest( | ||
event: SQSEvent, | ||
context: Context | ||
): String = { | ||
implicit val actorSystem: ActorSystem = | ||
ActorSystem("main-actor-system") | ||
implicit val ec: ExecutionContext = | ||
actorSystem.dispatcher | ||
val batchProcessor = BatchProcessor(config) | ||
|
||
def processEvent(event: SQSEvent): Future[String] = { | ||
info(s"running relation_embedder lambda, got event: $event") | ||
|
||
// Wait here so that lambda can finish executing correctly. | ||
// 15 minutes is the maximum time allowed for a lambda to run. | ||
Await.result( | ||
Future.sequence(event.extractBatches.map(batchProcessor(_))), | ||
15.minutes | ||
) | ||
"Done" | ||
Future.sequence(event.extractBatches.map(batchProcessor(_))).map(_ => "Done") | ||
} | ||
} |
32 changes: 32 additions & 0 deletions
32
...dder/relation_embedder/src/main/scala/weco/pipeline/relation_embedder/lib/LambdaApp.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
package weco.pipeline.relation_embedder.lib | ||
|
||
import com.amazonaws.services.lambda.runtime.{Context, RequestHandler} | ||
import grizzled.slf4j.Logging | ||
import org.apache.pekko.actor.ActorSystem | ||
|
||
import scala.concurrent.{Await, ExecutionContext, Future} | ||
import scala.concurrent.duration.{DurationInt, FiniteDuration} | ||
|
||
trait LambdaApp[In, Out, Config <: ApplicationConfig] | ||
extends RequestHandler[In, Out] | ||
with LambdaConfigurable[Config] | ||
with Logging { | ||
|
||
// 15 minutes is the maximum time allowed for a lambda to run, as of 2024-12-19 | ||
private val maximumExecutionTime: FiniteDuration = 15.minutes | ||
|
||
implicit val actorSystem: ActorSystem = | ||
ActorSystem("main-actor-system") | ||
implicit val ec: ExecutionContext = | ||
actorSystem.dispatcher | ||
|
||
def processEvent(in: In): Future[Out] | ||
|
||
override def handleRequest( | ||
event: In, | ||
context: Context | ||
): Out = Await.result( | ||
processEvent(event), | ||
maximumExecutionTime | ||
) | ||
} |