improve performance of scrapping movies

ratanparai · May 9, 2021 · ae24930 · ae24930
1 parent 285bf98
commit ae24930
Show file tree

Hide file tree

Showing 2 changed files with 51 additions and 41 deletions.
diff --git a/.gitignore b/.gitignore
@@ -24,4 +24,17 @@ app/.settings/org.eclipse.buildship.core.prefs
 .idea/codeStyles/codeStyleConfig.xml
 .idea/codeStyles/Project.xml
 
-apikey.properties
+apikey.properties
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+.idea/artifacts
+.idea/compiler.xml
+.idea/jarRepositories.xml
+.idea/modules.xml
+.idea/*.iml
+.idea/modules
+*.iml
+*.ipr
diff --git a/app/src/main/java/com/ratanparai/moviedog/service/MovieService.kt b/app/src/main/java/com/ratanparai/moviedog/service/MovieService.kt
@@ -13,8 +13,7 @@ import com.ratanparai.moviedog.db.entity.Scrapped
 import com.ratanparai.moviedog.db.entity.SearchHash
 import com.ratanparai.moviedog.scrapper.*
 import com.ratanparai.moviedog.utilities.MD5
-import kotlinx.coroutines.async
-import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.*
 
 class MovieService(private val context: Context) {
 
@@ -86,56 +85,54 @@ class MovieService(private val context: Context) {
 
             Log.d(TAG, "Scrapping ${movieLinks.size} movies!")
 
+            coroutineScope {
+                movieLinks.map { link -> async(Dispatchers.IO) {
+                    if (alreadyScrappedMovie(link, scrappedDao)) {
+                        Log.d(TAG, "Movie is already scrapped for URL: $link")
+                        return@async
+                    }
 
-            for (link in movieLinks) {
-
-                if (alreadyScrappedMovie(link, scrappedDao)) {
-                    Log.d(TAG, "Movie is already scrapped for URL: $link")
-                    continue
-                }
+                    Log.d(TAG, "First time scrapping movie for URL: $link")
 
-                Log.d(TAG, "First time scrapping movie for URL: $link")
+                    val scrapped = Scrapped(
+                        url = link
+                    )
 
-                val scrapped = Scrapped(
-                    url = link
-                )
+                    try {
+                        val movieDoc = scrapper.getDocument(link)
+                        val movie = scrapper.getMovie(movieDoc, link)
+                        Log.d(TAG, "Scrapped movie: $movie for search URL $searchUrl ")
 
-                try {
-                    val movieDoc = scrapper.getDocument(link)
-                    val movie = scrapper.getMovie(movieDoc, link)
-                    Log.d(TAG, "Scrapped movie: $movie for search URL $searchUrl ")
+                        scrappedDao.insert(scrapped)
 
-                    scrappedDao.insert(scrapped)
+                        val movieFromDao = movieDao.getMovieByTitle(movie.title)
+                        if (movieFromDao == null) {
+                            Log.d(TAG, "The movie is not in database. Inserting movie info and first video link")
+                            val movieId = movieDao.insertMovie(movie).toInt()
 
-                    val movieFromDao = movieDao.getMovieByTitle(movie.title)
-                    if (movieFromDao == null) {
-                        Log.d(TAG, "The movie is not in database. Inserting movie info and first video link")
-                        val movieId = movieDao.insertMovie(movie).toInt()
+                            val subtitleService = SubtitleService(context)
+                            val movieWithId = movie.copy(movieId)
+                            subtitleService.backgroundSubtitleDownload(movieWithId)
 
-                        val subtitleService = SubtitleService(context)
-                        val movieWithId = movie.copy(movieId)
-                        subtitleService.backgroundSubtitleDownload(movieWithId)
+                            val movieUrl = MovieUrl(movieId = movieId, movieUrl = movie.videoUrl, serviceName = serviceName)
+                            movieUrlDao.insertMovieUrl(movieUrl)
+                        } else {
+                            Log.d(TAG, "Movie is already in database. Adding new video urls")
+                            val movieUrl = MovieUrl(
+                                movieId = movieFromDao.id,
+                                movieUrl = movie.videoUrl,
+                                serviceName = serviceName
+                            )
 
-                        val movieUrl = MovieUrl(movieId = movieId, movieUrl = movie.videoUrl, serviceName = serviceName)
-                        movieUrlDao.insertMovieUrl(movieUrl)
-                    } else {
-                        Log.d(TAG, "Movie is already in database. Adding new video urls")
-                        val movieUrl = MovieUrl(
-                            movieId = movieFromDao.id,
-                            movieUrl = movie.videoUrl,
-                            serviceName = serviceName
-                        )
+                            movieUrlDao.insertMovieUrl(movieUrl)
+                        }
 
-                        movieUrlDao.insertMovieUrl(movieUrl)
+                    } catch (ex: Exception) {
+                        Log.d("MovieService", ex.message)
                     }
-
-                } catch (ex: Exception) {
-                    Log.d("MovieService", ex.message)
-                }
-
+                } }.awaitAll()
             }
 
-
             if (sHash == null) {
                 val searchHash = SearchHash(url = searchUrl, md5hash = md5Hex)
                 searchHashDao.add(searchHash)