diff --git a/app/src/main/java/com/dd3boh/outertune/MainActivity.kt b/app/src/main/java/com/dd3boh/outertune/MainActivity.kt index 5a42ca9c9..8efb0f758 100644 --- a/app/src/main/java/com/dd3boh/outertune/MainActivity.kt +++ b/app/src/main/java/com/dd3boh/outertune/MainActivity.kt @@ -104,11 +104,7 @@ import com.dd3boh.outertune.ui.screens.search.OnlineSearchScreen import com.dd3boh.outertune.ui.screens.settings.* import com.dd3boh.outertune.ui.theme.* import com.dd3boh.outertune.ui.utils.appBarScrollBehavior -import com.dd3boh.outertune.ui.utils.localToRemoteArtist -import com.dd3boh.outertune.ui.utils.quickSync import com.dd3boh.outertune.ui.utils.resetHeightOffset -import com.dd3boh.outertune.ui.utils.scanLocal -import com.dd3boh.outertune.ui.utils.unloadScanner import com.dd3boh.outertune.utils.SyncUtils import com.dd3boh.outertune.utils.dataStore import com.dd3boh.outertune.utils.get @@ -116,6 +112,8 @@ import com.dd3boh.outertune.utils.purgeCache import com.dd3boh.outertune.utils.rememberEnumPreference import com.dd3boh.outertune.utils.rememberPreference import com.dd3boh.outertune.utils.reportException +import com.dd3boh.outertune.utils.scanners.LocalMediaScanner +import com.dd3boh.outertune.utils.scanners.LocalMediaScanner.Companion.unloadAdvancedScanner import dagger.hilt.android.AndroidEntryPoint import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -235,18 +233,20 @@ class MainActivity : ComponentActivity() { val (autoScan) = rememberPreference(AutomaticScannerKey, defaultValue = true) if (autoScan) { + val scanner = LocalMediaScanner.getScanner() + // equivalent to (quick scan) - val directoryStructure = scanLocal(this, database, ScannerImpl.MEDIASTORE).value - quickSync( + val directoryStructure = scanner.scanLocal(this, database, ScannerImpl.MEDIASTORE).value + scanner.quickSync( database, directoryStructure.toList(), scannerSensitivity, strictExtensions, scannerType ) - unloadScanner() + unloadAdvancedScanner() // start artist linking job if (lookupYtmArtists) { CoroutineScope(Dispatchers.IO).launch { - localToRemoteArtist(database) + scanner.localToRemoteArtist(database) } } purgeCache() // juuuust to be sure diff --git a/app/src/main/java/com/dd3boh/outertune/models/DirectoryTree.kt b/app/src/main/java/com/dd3boh/outertune/models/DirectoryTree.kt new file mode 100644 index 000000000..b8640079d --- /dev/null +++ b/app/src/main/java/com/dd3boh/outertune/models/DirectoryTree.kt @@ -0,0 +1,179 @@ +package com.dd3boh.outertune.models + +import com.dd3boh.outertune.db.entities.Song +import com.dd3boh.outertune.ui.utils.SCANNER_DEBUG +import com.dd3boh.outertune.ui.utils.sdcardRoot +import timber.log.Timber + +/** + * A tree representation of local audio files + * + * @param path root directory start + */ +class DirectoryTree(path: String) { + companion object { + const val TAG = "DirectoryTree" + var directoryUID = 0 + } + + /** + * Directory name + */ + var currentDir = path // file name + + /** + * Full parent directory path + */ + var parent: String = "" + + // folder contents + var subdirs = ArrayList() + var files = ArrayList() + + val uid = directoryUID + + init { + // increment uid + directoryUID++ + } + + /** + * Instantiate a directory tree directly + */ + constructor(path: String, files: ArrayList) : this(path) { + this.files = files + } + + fun insert(path: String, song: Song) { +// println("curr path =" + path) + + // add a file + if (path.indexOf('/') == -1) { + files.add(song) + if (SCANNER_DEBUG) + Timber.tag(TAG).d("Adding song with path: $path") + return + } + + // there is still subdirs to process + var tmpPath = path + if (path[path.length - 1] == '/') { + tmpPath = path.substring(0, path.length - 1) + } + + // the first directory before the . + val subdirPath = tmpPath.substringBefore('/') + + // create subdirs if they do not exist, then insert + var existingSubdir: DirectoryTree? = null + subdirs.forEach { subdir -> + if (subdir.currentDir == subdirPath) { + existingSubdir = subdir + return@forEach + } + } + + if (existingSubdir == null) { + val tree = DirectoryTree(subdirPath) + tree.parent = "$parent/$currentDir" + tree.insert(tmpPath.substringAfter('/'), song) + subdirs.add(tree) + + } else { + existingSubdir!!.insert(tmpPath.substringAfter('/'), song) + } + } + + + /** + * Get the name of the file from full path, without any extensions + */ + private fun getFileName(path: String?): String? { + if (path == null) { + return null + } + return path.substringAfterLast('/').substringBefore('.') + } + + /** + * Retrieves song object at path + * + * @return song at path, or null if it does not exist + */ + fun getSong(path: String): Song? { + Timber.tag(TAG).d("Searching for song, at path: $path") + + // search for song in current dir + if (path.indexOf('/') == -1) { + val foundSong: Song = files.first { getFileName(it.song.localPath) == getFileName(path) } + Timber.tag(TAG).d("Searching for song, found?: ${foundSong.id} Name: ${foundSong.song.title}") + return foundSong + } + + // there is still subdirs to process + var tmpPath = path + if (path[path.length - 1] == '/') { + tmpPath = path.substring(0, path.length - 1) + } + + // the first directory before the . + val subdirPath = tmpPath.substringBefore('/') + + // scan for matching subdirectory + var existingSubdir: DirectoryTree? = null + subdirs.forEach { subdir -> + if (subdir.currentDir == subdirPath) { + existingSubdir = subdir + return@forEach + } + } + + // explore the subdirectory if it exists in + if (existingSubdir == null) { + return null + } else { + return existingSubdir!!.getSong(tmpPath.substringAfter('/')) + } + } + + + /** + * Retrieve a list of all the songs + */ + fun toList(): List { + val songs = ArrayList() + + fun traverseTree(tree: DirectoryTree, result: ArrayList) { + result.addAll(tree.files) + tree.subdirs.forEach { traverseTree(it, result) } + } + + traverseTree(this, songs) + return songs + } + + /** + * Retrieves a modified version of this DirectoryTree. + * All folders are recognized to be top level folders + */ + fun toFlattenedTree(): DirectoryTree { + val result = DirectoryTree(sdcardRoot) + getSubdirsRecursive(this, result.subdirs) + return result + } + + /** + * Crawl the directory tree, add the subdirectories with songs to the list + * @param it + * @param result + */ + private fun getSubdirsRecursive(it: DirectoryTree, result: ArrayList) { + if (it.files.size > 0) { + result.add(DirectoryTree(it.currentDir, it.files)) + } + + if (it.subdirs.size > 0) { + it.subdirs.forEach { getSubdirsRecursive(it, result) } + } + } +} \ No newline at end of file diff --git a/app/src/main/java/com/dd3boh/outertune/models/SongTempData.kt b/app/src/main/java/com/dd3boh/outertune/models/SongTempData.kt new file mode 100644 index 000000000..12064ada4 --- /dev/null +++ b/app/src/main/java/com/dd3boh/outertune/models/SongTempData.kt @@ -0,0 +1,11 @@ +package com.dd3boh.outertune.models + +import com.dd3boh.outertune.db.entities.FormatEntity + +/** + * For passing along song metadata + */ +data class SongTempData( + val id: String, val path: String, val title: String, val duration: Int, val artist: String?, + val artistID: String?, val album: String?, val albumID: String?, val formatEntity: FormatEntity, +) \ No newline at end of file diff --git a/app/src/main/java/com/dd3boh/outertune/ui/component/Items.kt b/app/src/main/java/com/dd3boh/outertune/ui/component/Items.kt index 19978598f..19fefe836 100644 --- a/app/src/main/java/com/dd3boh/outertune/ui/component/Items.kt +++ b/app/src/main/java/com/dd3boh/outertune/ui/component/Items.kt @@ -102,7 +102,7 @@ import com.dd3boh.outertune.models.MediaMetadata import com.dd3boh.outertune.playback.queues.ListQueue import com.dd3boh.outertune.ui.menu.FolderMenu import com.dd3boh.outertune.ui.theme.extractThemeColor -import com.dd3boh.outertune.ui.utils.DirectoryTree +import com.dd3boh.outertune.models.DirectoryTree import com.dd3boh.outertune.ui.utils.getLocalThumbnail import com.dd3boh.outertune.utils.joinByBullet import com.dd3boh.outertune.utils.makeTimeString diff --git a/app/src/main/java/com/dd3boh/outertune/ui/menu/FolderMenu.kt b/app/src/main/java/com/dd3boh/outertune/ui/menu/FolderMenu.kt index cc9bba444..a760a946a 100644 --- a/app/src/main/java/com/dd3boh/outertune/ui/menu/FolderMenu.kt +++ b/app/src/main/java/com/dd3boh/outertune/ui/menu/FolderMenu.kt @@ -26,7 +26,7 @@ import com.dd3boh.outertune.extensions.toMediaItem import com.dd3boh.outertune.ui.component.GridMenu import com.dd3boh.outertune.ui.component.GridMenuItem import com.dd3boh.outertune.ui.component.SongFolderItem -import com.dd3boh.outertune.ui.utils.DirectoryTree +import com.dd3boh.outertune.models.DirectoryTree @Composable fun FolderMenu( diff --git a/app/src/main/java/com/dd3boh/outertune/ui/screens/settings/LocalPlayerSettings.kt b/app/src/main/java/com/dd3boh/outertune/ui/screens/settings/LocalPlayerSettings.kt index 0405cb4ff..cde6b5cee 100644 --- a/app/src/main/java/com/dd3boh/outertune/ui/screens/settings/LocalPlayerSettings.kt +++ b/app/src/main/java/com/dd3boh/outertune/ui/screens/settings/LocalPlayerSettings.kt @@ -64,15 +64,11 @@ import com.dd3boh.outertune.ui.component.PreferenceGroupTitle import com.dd3boh.outertune.ui.component.SwitchPreference import com.dd3boh.outertune.ui.utils.backToMain -import com.dd3boh.outertune.ui.utils.localToRemoteArtist -import com.dd3boh.outertune.ui.utils.quickSync -import com.dd3boh.outertune.ui.utils.refreshLocal -import com.dd3boh.outertune.ui.utils.scanLocal -import com.dd3boh.outertune.ui.utils.syncDB -import com.dd3boh.outertune.ui.utils.unloadScanner import com.dd3boh.outertune.utils.purgeCache import com.dd3boh.outertune.utils.rememberEnumPreference import com.dd3boh.outertune.utils.rememberPreference +import com.dd3boh.outertune.utils.scanners.LocalMediaScanner.Companion.getScanner +import com.dd3boh.outertune.utils.scanners.LocalMediaScanner.Companion.unloadAdvancedScanner import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import timber.log.Timber @@ -177,31 +173,32 @@ fun LocalPlayerSettings( Toast.LENGTH_SHORT ).show() coroutineScope.launch(Dispatchers.IO) { + val scanner = getScanner() // full rescan if (fullRescan) { - val directoryStructure = scanLocal(context, database, scannerType).value - syncDB(database, directoryStructure.toList(), scannerSensitivity, strictExtensions, true) - unloadScanner() + val directoryStructure = scanner.scanLocal(context, database, scannerType).value + scanner.syncDB(database, directoryStructure.toList(), scannerSensitivity, strictExtensions, true) + unloadAdvancedScanner() // start artist linking job if (lookupYtmArtists) { coroutineScope.launch(Dispatchers.IO) { - localToRemoteArtist(database) + scanner.localToRemoteArtist(database) } } } else { // quick scan - val directoryStructure = scanLocal(context, database, ScannerImpl.MEDIASTORE).value - quickSync( + val directoryStructure = scanner.scanLocal(context, database, ScannerImpl.MEDIASTORE).value + scanner.quickSync( database, directoryStructure.toList(), scannerSensitivity, strictExtensions, scannerType ) - unloadScanner() + unloadAdvancedScanner() // start artist linking job if (lookupYtmArtists) { coroutineScope.launch(Dispatchers.IO) { - localToRemoteArtist(database) + scanner.localToRemoteArtist(database) } } } diff --git a/app/src/main/java/com/dd3boh/outertune/ui/utils/LocalMediaUtils.kt b/app/src/main/java/com/dd3boh/outertune/ui/utils/LocalMediaUtils.kt index 559726cb7..e8244f9d5 100644 --- a/app/src/main/java/com/dd3boh/outertune/ui/utils/LocalMediaUtils.kt +++ b/app/src/main/java/com/dd3boh/outertune/ui/utils/LocalMediaUtils.kt @@ -1,51 +1,20 @@ package com.dd3boh.outertune.ui.utils -import android.content.ContentResolver -import android.content.Context import android.graphics.Bitmap import android.graphics.BitmapFactory import android.media.MediaMetadataRetriever -import android.os.Build import android.provider.MediaStore -import com.dd3boh.outertune.constants.ScannerMatchCriteria -import com.dd3boh.outertune.constants.ScannerImpl -import com.dd3boh.outertune.db.MusicDatabase -import com.dd3boh.outertune.db.entities.AlbumEntity -import com.dd3boh.outertune.db.entities.ArtistEntity -import com.dd3boh.outertune.db.entities.FormatEntity -import com.dd3boh.outertune.db.entities.Song -import com.dd3boh.outertune.db.entities.SongArtistMap -import com.dd3boh.outertune.db.entities.SongEntity -import com.dd3boh.outertune.models.toMediaMetadata +import com.dd3boh.outertune.models.DirectoryTree import com.dd3boh.outertune.utils.cache import com.dd3boh.outertune.utils.retrieveImage -import com.dd3boh.outertune.utils.scanners.ExtraMetadataWrapper -import com.dd3boh.outertune.utils.scanners.FFProbeScanner -import com.dd3boh.outertune.utils.scanners.MetadataScanner -import com.zionhuang.innertube.YouTube -import com.zionhuang.innertube.YouTube.search -import kotlinx.coroutines.Deferred import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.ExperimentalCoroutinesApi -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.flow.MutableStateFlow -import kotlinx.coroutines.flow.first -import kotlinx.coroutines.flow.firstOrNull -import kotlinx.coroutines.runBlocking import timber.log.Timber -import java.io.File -import java.lang.Integer.parseInt -import java.time.LocalDateTime -import java.util.Locale const val TAG = "LocalMediaUtils" /** - * TODO: Implement a proper, much faster scanner - * Currently ffmpeg-kit will fail if you hit it with too many calls too quickly. - * You can try more than 8 jobs, but good luck. - * For easier debugging, uncomment SCANNER_CRASH_AT_FIRST_ERROR to stop at first error + * For easier debugging, set SCANNER_CRASH_AT_FIRST_ERROR to stop at first error */ const val SCANNER_CRASH_AT_FIRST_ERROR = false // crash at ffprobe errors only const val SYNC_SCANNER = false // true will not use multithreading for scanner @@ -59,8 +28,7 @@ val scannerSession = Dispatchers.IO.limitedParallelism(MAX_CONCURRENT_JOBS) const val sdcardRoot = "/storage/emulated/0/" val testScanPaths = arrayListOf("Music") val ARTIST_SEPARATORS = Regex("\\s*;\\s*|\\s*ft\\.\\s*|\\s*feat\\.\\s*|\\s*&\\s*", RegexOption.IGNORE_CASE) -var directoryUID = 0 -var cachedDirectoryTree: DirectoryTree? = null +private var cachedDirectoryTree: DirectoryTree? = null // useful metadata @@ -80,952 +48,6 @@ val projection = arrayOf( ) -/** - * A tree representation of local audio files - * - * @param path root directory start - */ -class DirectoryTree(path: String) { - companion object { - const val TAG = "DirectoryTree" - } - - /** - * Directory name - */ - var currentDir = path // file name - - /** - * Full parent directory path - */ - var parent: String = "" - - // folder contents - var subdirs = ArrayList() - var files = ArrayList() - - val uid = directoryUID - - init { - // increment uid - directoryUID++ - } - - /** - * Instantiate a directory tree directly - */ - constructor(path: String, files: ArrayList) : this(path) { - this.files = files - } - - fun insert(path: String, song: Song) { -// println("curr path =" + path) - - // add a file - if (path.indexOf('/') == -1) { - files.add(song) - if (SCANNER_DEBUG) - Timber.tag(TAG).d("Adding song with path: $path") - return - } - - // there is still subdirs to process - var tmpPath = path - if (path[path.length - 1] == '/') { - tmpPath = path.substring(0, path.length - 1) - } - - // the first directory before the . - val subdirPath = tmpPath.substringBefore('/') - - // create subdirs if they do not exist, then insert - var existingSubdir: DirectoryTree? = null - subdirs.forEach { subdir -> - if (subdir.currentDir == subdirPath) { - existingSubdir = subdir - return@forEach - } - } - - if (existingSubdir == null) { - val tree = DirectoryTree(subdirPath) - tree.parent = "$parent/$currentDir" - tree.insert(tmpPath.substringAfter('/'), song) - subdirs.add(tree) - - } else { - existingSubdir!!.insert(tmpPath.substringAfter('/'), song) - } - } - - - /** - * Get the name of the file from full path, without any extensions - */ - private fun getFileName(path: String?): String? { - if (path == null) { - return null - } - return path.substringAfterLast('/').substringBefore('.') - } - - /** - * Retrieves song object at path - * - * @return song at path, or null if it does not exist - */ - fun getSong(path: String): Song? { - Timber.tag(TAG).d("Searching for song, at path: $path") - - // search for song in current dir - if (path.indexOf('/') == -1) { - val foundSong: Song = files.first { getFileName(it.song.localPath) == getFileName(path) } - Timber.tag(TAG).d("Searching for song, found?: ${foundSong.id} Name: ${foundSong.song.title}") - return foundSong - } - - // there is still subdirs to process - var tmpPath = path - if (path[path.length - 1] == '/') { - tmpPath = path.substring(0, path.length - 1) - } - - // the first directory before the . - val subdirPath = tmpPath.substringBefore('/') - - // scan for matching subdirectory - var existingSubdir: DirectoryTree? = null - subdirs.forEach { subdir -> - if (subdir.currentDir == subdirPath) { - existingSubdir = subdir - return@forEach - } - } - - // explore the subdirectory if it exists in - if (existingSubdir == null) { - return null - } else { - return existingSubdir!!.getSong(tmpPath.substringAfter('/')) - } - } - - - /** - * Retrieve a list of all the songs - */ - fun toList(): List { - val songs = ArrayList() - - fun traverseTree(tree: DirectoryTree, result: ArrayList) { - result.addAll(tree.files) - tree.subdirs.forEach { traverseTree(it, result) } - } - - traverseTree(this, songs) - return songs - } - - /** - * Retrieves a modified version of this DirectoryTree. - * All folders are recognized to be top level folders - */ - fun toFlattenedTree(): DirectoryTree { - val result = DirectoryTree(sdcardRoot) - getSubdirsRecursive(this, result.subdirs) - return result - } - - /** - * Crawl the directory tree, add the subdirectories with songs to the list - * @param it - * @param result - */ - private fun getSubdirsRecursive(it: DirectoryTree, result: ArrayList) { - if (it.files.size > 0) { - result.add(DirectoryTree(it.currentDir, it.files)) - } - - if (it.subdirs.size > 0) { - it.subdirs.forEach { getSubdirsRecursive(it, result) } - } - } -} - - -/** - * ========================== - * Actual local scanner utils - * ========================== - */ - - -var advancedScannerImpl: MetadataScanner? = null - -/** - * TODO: Docs here - */ -fun getScanner(scannerImpl: ScannerImpl): MetadataScanner? { - // kotlin won't let me return MetadataScanner even if it cant possibly be null broooo - return when (scannerImpl) { - ScannerImpl.FFPROBE, ScannerImpl.MEDIASTORE_FFPROBE -> - if (advancedScannerImpl is FFProbeScanner) advancedScannerImpl else FFProbeScanner() - ScannerImpl.MEDIASTORE -> null - } -} - -fun unloadScanner() { - advancedScannerImpl = null -} - -/** - * Dev uses - */ -fun refreshLocal(context: Context, database: MusicDatabase) = - refreshLocal(context, database, testScanPaths) - - -/** - * Quickly rebuild a skeleton directory tree of local files based on the database - * - * Notes: - * If files move around, that's on you to re run the scanner. - * If the metadata changes, that's also on you to re run the scanner. - * - * @param context Context - * @param scanPaths List of whitelist paths to scan under. This assumes - * the current directory is /storage/emulated/0/ a.k.a, /sdcard. - * For example, to scan under Music and Documents/songs --> ("Music", Documents/songs) - */ -fun refreshLocal( - context: Context, - database: MusicDatabase, - scanPaths: ArrayList -): MutableStateFlow { - val newDirectoryStructure = DirectoryTree(sdcardRoot) - - // get songs from db - var existingSongs: List - runBlocking(Dispatchers.IO) { - existingSongs = database.allLocalSongs().first() - } - - // Query for audio files - val contentResolver: ContentResolver = context.contentResolver - val cursor = contentResolver.query( - MediaStore.Audio.Media.EXTERNAL_CONTENT_URI, - projection, - "${MediaStore.Audio.Media.IS_MUSIC} != 0 AND ${MediaStore.Audio.Media.DATA} LIKE ?", - scanPaths.map { "$sdcardRoot$it%" }.toTypedArray(), // whitelist paths - null - ) - Timber.tag(TAG).d("------------ SCAN: Starting Quick Directory Rebuild ------------") - cursor?.use { localCursor -> - // Columns indices - val nameColumn = localCursor.getColumnIndexOrThrow(MediaStore.Audio.Media.DISPLAY_NAME) - val pathColumn = localCursor.getColumnIndexOrThrow(MediaStore.Audio.Media.RELATIVE_PATH) - - while (localCursor.moveToNext()) { - val name = localCursor.getString(nameColumn) // file name - val path = localCursor.getString(pathColumn) - - if (SCANNER_DEBUG) - Timber.tag(TAG).d("Quick scanner: PATH: $path") - - // Build directory tree with existing files - val possibleMatch = existingSongs.firstOrNull() { it.song.localPath == "$sdcardRoot$path$name" } - - if (possibleMatch != null) { - newDirectoryStructure.insert("$path$name", possibleMatch) - } - - } - } - - Timber.tag(TAG).d("------------ SCAN: Finished Quick Directory Rebuild ------------") - cachedDirectoryTree = newDirectoryStructure - return MutableStateFlow(newDirectoryStructure) -} - -/** - * For passing along song metadata - */ -data class SongTempData( - val id: String, val path: String, val title: String, val duration: Int, val artist: String?, - val artistID: String?, val album: String?, val albumID: String?, val formatEntity: FormatEntity, -) - -/** - * Compiles a song with all it's necessary metadata. Unlike MediaStore, - * this also supports multiple artists, multiple genres (TBD), and a few extra details (TBD). - */ -fun advancedScan( - basicData: SongTempData, - database: MusicDatabase, - scannerImpl: ScannerImpl, -): Song { - val artists = ArrayList() -// var generes -// var year: String? = null - - // MediaStore mode - var rawArtists = basicData.artist - - try { - // decide which scanner to use - val scanner = getScanner(scannerImpl) - var ffmpegData: ExtraMetadataWrapper? = null - if (scannerImpl == ScannerImpl.MEDIASTORE_FFPROBE) { - ffmpegData = scanner?.getMediaStoreSupplement(basicData.path) - rawArtists = ffmpegData?.artists - } else if (scannerImpl == ScannerImpl.FFPROBE){ - ffmpegData = scanner?.getAllMetadata(basicData.path, basicData.formatEntity) - rawArtists = ffmpegData?.artists - } - - // parse data - rawArtists?.split(ARTIST_SEPARATORS)?.forEach { element -> - val artistVal = element.trim() - artists.add(ArtistEntity("LA${ArtistEntity.generateArtistId()}", artistVal, isLocal = true)) - } - - // file format info - if (scannerImpl == ScannerImpl.FFPROBE && ffmpegData?.format != null) { - database.query { - upsert( - ffmpegData.format!! - ) - } - } else { // MEDIASTORE_FFPROBE and MEDIASTORE - database.query { - upsert( - basicData.formatEntity - ) - } - } - } catch (e: Exception) { - if (SCANNER_CRASH_AT_FIRST_ERROR) { - throw Exception("HALTING AT FIRST SCANNER ERROR " + e.message) // debug - } - // fallback on media store - if (SCANNER_DEBUG) { - Timber.tag(TAG).d( - "ERROR READING ARTIST METADATA: ${e.message}" + - " Falling back on MediaStore for ${basicData.path}" - ) - e.printStackTrace() - } - - if (basicData.artist?.isNotBlank() == true) { - if (SCANNER_DEBUG) { - Timber.tag(TAG).d( - "Adding local artist with name: ${basicData.artist}" - ) - artists.add(ArtistEntity(ArtistEntity.generateArtistId(), basicData.artist, isLocal = true)) - } - } - } - - return Song( - SongEntity( - basicData.id, - basicData.title, - (basicData.duration / 1000), // we use seconds for duration - albumId = basicData.albumID, - albumName = basicData.album, - isLocal = true, - inLibrary = LocalDateTime.now(), - localPath = basicData.path - ), - artists, - // album not working - basicData.albumID?.let { - basicData.album?.let { it1 -> - AlbumEntity( - it, - title = it1, - duration = 0, - songCount = 1 - ) - } - } - ) -} - -/** - * Dev uses - */ -fun scanLocal(context: Context, database: MusicDatabase, scannerImpl: ScannerImpl) = - scanLocal(context, database, testScanPaths, scannerImpl) - -/** - * Scan MediaStore for songs given a list of paths to scan for. - * This will replace all data in the database for a given song. - * - * @param context Context - * @param scanPaths List of whitelist paths to scan under. This assumes - * the current directory is /storage/emulated/0/ a.k.a, /sdcard. - * For example, to scan under Music and Documents/songs --> ("Music", Documents/songs) - */ -@OptIn(ExperimentalCoroutinesApi::class) -fun scanLocal( - context: Context, - database: MusicDatabase, - scanPaths: ArrayList, - scannerImpl: ScannerImpl, -): MutableStateFlow { - - val newDirectoryStructure = DirectoryTree(sdcardRoot) - val contentResolver: ContentResolver = context.contentResolver - - // Query for audio files - val cursor = contentResolver.query( - MediaStore.Audio.Media.EXTERNAL_CONTENT_URI, - projection, - "${MediaStore.Audio.Media.IS_MUSIC} != 0 AND ${MediaStore.Audio.Media.DATA} LIKE ?", - scanPaths.map { "$sdcardRoot$it%" }.toTypedArray(), // whitelist paths - null - ) - Timber.tag(TAG).d("------------ SCAN: Starting Full Scanner ------------") - - val scannerJobs = ArrayList>() - runBlocking { - // MediaStore is our "basic" scanner & file discovery - cursor?.use { cursor -> - // Columns indices - val nameColumn = cursor.getColumnIndexOrThrow(MediaStore.Audio.Media.DISPLAY_NAME) - val titleColumn = cursor.getColumnIndexOrThrow(MediaStore.Audio.Media.TITLE) - val durationColumn = cursor.getColumnIndexOrThrow(MediaStore.Audio.Media.DURATION) - val artistIdColumn = cursor.getColumnIndexOrThrow(MediaStore.Audio.Media.ALBUM_ID) - val artistColumn = cursor.getColumnIndexOrThrow(MediaStore.Audio.Media.ARTIST) - val albumIDColumn = cursor.getColumnIndexOrThrow(MediaStore.Audio.Media.ALBUM_ID) - val albumColumn = cursor.getColumnIndexOrThrow(MediaStore.Audio.Media.ALBUM) - val pathColumn = cursor.getColumnIndexOrThrow(MediaStore.Audio.Media.RELATIVE_PATH) - - val mimeColumn = cursor.getColumnIndexOrThrow(MediaStore.Audio.Media.MIME_TYPE) - val bitrateColumn = cursor.getColumnIndexOrThrow(MediaStore.Audio.Media.BITRATE) - - while (cursor.moveToNext()) { - val id = SongEntity.generateSongId() - val name = cursor.getString(nameColumn) // file name - val title = cursor.getString(titleColumn) // song title - val duration = cursor.getInt(durationColumn) - val artist = cursor.getString(artistColumn) - val artistID = cursor.getString(artistIdColumn) - val albumID = cursor.getString(albumIDColumn) - val album = cursor.getString(albumColumn) - val path = cursor.getString(pathColumn) - - // extra stream info - val bitrate = cursor.getInt(bitrateColumn) - val mime = cursor.getString(mimeColumn) - - if (SCANNER_DEBUG) - Timber.tag(TAG).d("ID: $id, Name: $name, ARTIST: $artist, PATH: $path") - - // append song to list - // media store doesn't support multi artists... - // do not link album (and whatever song id) with youtube yet, figure that out later - - if (!SYNC_SCANNER) { - // use async scanner - scannerJobs.add( - async(scannerSession) { - advancedScan( - SongTempData( - id.toString(), "$sdcardRoot$path$name", title, duration, - artist, artistID, album, albumID, - FormatEntity( - id = id, - itag = -1, - mimeType = mime, - codecs = mime.substringAfter('/'), - bitrate = bitrate, - sampleRate = -1, - contentLength = duration.toLong(), - loudnessDb = null, - playbackUrl = null - ), - ), database, scannerImpl, - ) - } - ) - } else { - // force synchronous scanning of songs - val toInsert = advancedScan( - SongTempData( - id.toString(), "$sdcardRoot$path$name", title, duration, - artist, artistID, album, albumID, - FormatEntity( - id = id, - itag = -1, - mimeType = mime, - codecs = mime.substringAfter('/'), - bitrate = bitrate, - sampleRate = -1, - contentLength = duration.toLong(), - loudnessDb = null, - playbackUrl = null - ) - ), database, scannerImpl - ) - toInsert.song.localPath?.let { s -> - newDirectoryStructure.insert( - s.substringAfter(sdcardRoot), toInsert - ) - } - } - } - } - - if (!SYNC_SCANNER) { - // use async scanner - scannerJobs.awaitAll() - } - } - - // build the tree - scannerJobs.forEach { - val song = it.getCompleted() - - song.song.localPath?.let { s -> - newDirectoryStructure.insert( - s.substringAfter(sdcardRoot), song - ) - } - } - - Timber.tag(TAG).d("------------ SCAN: Finished Full Scanner ------------") - cachedDirectoryTree = newDirectoryStructure - return MutableStateFlow(newDirectoryStructure) -} - -/** - * Search for an artist on YouTube Music. - * - * If no artist is found, create one locally - */ -fun youtubeArtistLookup(query: String): ArtistEntity? { - var ytmResult: ArtistEntity? = null - - // hit up YouTube for artist - runBlocking(Dispatchers.IO) { - search(query, YouTube.SearchFilter.FILTER_ARTIST).onSuccess { result -> - - val foundArtist = result.items.firstOrNull { - it.title.lowercase(Locale.getDefault()) == query.lowercase(Locale.getDefault()) - } ?: throw Exception("Failed to search: Artist not found on YouTube Music") - ytmResult = ArtistEntity( - foundArtist.id, - foundArtist.title, - foundArtist.thumbnail - ) - - if (SCANNER_DEBUG) - Timber.tag(TAG).d("Found remote artist: ${result.items.first().title}") - }.onFailure { - throw Exception("Failed to search on YouTube Music") - } - - } - - return ytmResult -} - -/** - * Update the Database with local files - * - * @param database - * @param newSongs - * @param matchStrength How lax should the scanner be - * @param strictFileNames Whether to consider file names - * @param refreshExisting Setting this this to true will updated existing songs - * with new information, else existing song's data will not be touched, regardless - * whether it was actually changed on disk - * - * Inserts a song if not found - * Updates a song information depending on if refreshExisting value - */ -fun syncDB( - database: MusicDatabase, - newSongs: List, - matchStrength: ScannerMatchCriteria, - strictFileNames: Boolean, - refreshExisting: Boolean? = false -) { - Timber.tag(TAG).d("------------ SYNC: Starting Local Library Sync ------------") - Timber.tag(TAG).d("Entries to process: ${newSongs.size}") - - newSongs.forEach { song -> - val querySong = database.searchSongsInclNotInLibrary(song.song.title) - - runBlocking(Dispatchers.IO) { - - // check if this song is known to the library - val songMatch = querySong.first().filter { - return@filter compareSong(it, song, matchStrength, strictFileNames) - } - - if (SCANNER_DEBUG) { - Timber.tag(TAG) - .d("Found songs that match: ${songMatch.size}, Total results from database: ${querySong.first().size}") - if (songMatch.isNotEmpty()) { - Timber.tag(TAG).d("FIRST Found songs ${songMatch.first().song.title}") - } - } - - - if (songMatch.isNotEmpty() && refreshExisting == true) { // known song, update the song info in the database - Timber.tag(TAG).d("Found in database, updating song: ${song.song.title}") - val songToUpdate = songMatch.first() - database.update(songToUpdate.song) - - // destroy existing artist links - database.unlinkSongArtists(songToUpdate.id) - - // update artists - var artistPos = 0 - song.artists.forEach { - val dbArtist = database.searchArtists(it.name).firstOrNull()?.firstOrNull() - - if (dbArtist == null) { - // artist does not exist in db, add it then link it - database.insert(it) - database.insert(SongArtistMap(songToUpdate.id, it.id, artistPos)) - } else { - // artist does exist in db, link to it - database.insert(SongArtistMap(songToUpdate.id, dbArtist.artist.id, artistPos)) - } - - artistPos++ - } - } else if (songMatch.isEmpty()) { // new song - if (SCANNER_DEBUG) - Timber.tag(TAG).d("NOT found in database, adding song: ${song.song.title}") - database.insert(song.toMediaMetadata()) - } - // do not delete songs from database automatically, we just disable them - disableSongs(database) - } - } - Timber.tag(TAG).d("------------ SYNC: Finished Local Library Sync ------------") -} - -/** - * A faster scanner implementation that adds new songs to the database, - * and does not touch older songs entires (apart from removing - * inacessable songs from libaray). - * - * No remote artist lookup is done - * - * WARNING: cachedDirectoryTree is not refreshed and may lead to inconsistencies. - * It is highly recommend to rebuild the tree after scanner operation - * - * @param newSongs List of songs. This is expecting a barebones DirectoryTree - * (only paths are necessary), thus you may use the output of refreshLocal().toList() - */ -@OptIn(ExperimentalCoroutinesApi::class) -fun quickSync( - database: MusicDatabase, - newSongs: List, - matchCriteria: ScannerMatchCriteria, - strictFileNames: Boolean, - scannerImpl: ScannerImpl, -) { - Timber.tag(TAG).d("------------ SYNC: Starting Quick (additive delta) Library Sync ------------") - Timber.tag(TAG).d("Entries to process: ${newSongs.size}") - - val mData = MediaMetadataRetriever() - - runBlocking(Dispatchers.IO) { - // get list of all songs in db, then get songs unknown to the database - val allSongs = database.allLocalSongs().first() - val delta = newSongs.filterNot { - allSongs.any { dbSong -> compareSong(it, dbSong, matchCriteria, strictFileNames) } - } - - val artistsWithMetadata = ArrayList() - val scannerJobs = ArrayList>() - runBlocking { - // Get song basic metadata - delta.forEach { s -> - mData.setDataSource(s.song.localPath) - - val id = SongEntity.generateSongId() - val title = - mData.extractMetadata(MediaMetadataRetriever.METADATA_KEY_TITLE).let { it ?: "" } // song title - val duration = parseInt(mData.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION)!!) - val artist = mData.extractMetadata(MediaMetadataRetriever.METADATA_KEY_ARTIST) - val artistID = if (artist == null) ArtistEntity.generateArtistId() else null - val album = mData.extractMetadata(MediaMetadataRetriever.METADATA_KEY_ALBUM) - val albumID = if (album == null) AlbumEntity.generateAlbumId() else null - // path should never be null since its coming from directory tree scanner - // but Kotlin is too dumb to care. Just ruthlessly suppress the error... - val path = "" + s.song.localPath - - // extra stream info - val bitrate = mData.extractMetadata(MediaMetadataRetriever.METADATA_KEY_BITRATE)?.let { parseInt(it) } - val mime = "" + mData.extractMetadata(MediaMetadataRetriever.METADATA_KEY_MIMETYPE) - var sampleRate = -1 - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { - sampleRate = mData.extractMetadata(MediaMetadataRetriever.METADATA_KEY_SAMPLERATE)?.let { parseInt(it) }!! - } - - - if (SCANNER_DEBUG) - Timber.tag(TAG).d("ID: $id, Title: $title, ARTIST: $artist, PATH: $path") - - // append song to list - // media store doesn't support multi artists... - // do not link album (and whatever song id) with youtube yet, figure that out later - - if (!SYNC_SCANNER) { - // use async scanner - scannerJobs.add( - async(scannerSession) { - advancedScan( - SongTempData( - id, path, title, duration, artist, artistID, album, albumID, - FormatEntity( - id = id, - itag = -1, - mimeType = mime, - codecs = mime.substringAfter('/'), - bitrate = bitrate?: -1, - sampleRate = sampleRate, - contentLength = duration.toLong(), - loudnessDb = null, - playbackUrl = null - ) - ), database, scannerImpl // no online artist lookup - ) - } - ) - } else { - // force synchronous scanning of songs - val toInsert = advancedScan( - SongTempData( - id, path, title, duration, artist, artistID, album, albumID, - FormatEntity( - id = id, - itag = -1, - mimeType = mime, - codecs = mime.substringAfter('/'), - bitrate = bitrate?: -1, - sampleRate = sampleRate, - contentLength = duration.toLong(), - loudnessDb = null, - playbackUrl = null - ) - ), database, scannerImpl - ) - artistsWithMetadata.add(toInsert) - } - } - } - - if (!SYNC_SCANNER) { - // use async scanner - scannerJobs.awaitAll() - } - - // add to finished list - scannerJobs.forEach { - artistsWithMetadata.add(it.getCompleted()) - } - - if (delta.isNotEmpty()) { - syncDB(database, artistsWithMetadata, matchCriteria, strictFileNames) - } - - disableSongs(database) - } - Timber.tag(TAG).d("------------ SYNC: Finished Quick (additive delta) Library Sync ------------") -} - -/** - * Converts all local artists to remote artists if possible - */ -fun localToRemoteArtist(database: MusicDatabase) { - runBlocking(Dispatchers.IO) { - val allLocal = database.allLocalArtists().first() - val scannerJobs = ArrayList>() - - allLocal.forEach { element -> - val artistVal = element.name.trim() - - // check if this artist exists in DB already - val databaseArtistMatch = - runBlocking(Dispatchers.IO) { - database.searchArtists(artistVal).first().filter { artist -> - // only look for remote artists here - return@filter artist.artist.name == artistVal && !artist.artist.isLocalArtist - } - } - - if (SCANNER_DEBUG) - Timber.tag(TAG).d("ARTIST FOUND IN DB??? Results size: ${databaseArtistMatch.size}") - - scannerJobs.add( - async(scannerSession) { - // resolve artist from YTM if not found in DB - if (databaseArtistMatch.isEmpty()) { - try { - youtubeArtistLookup(artistVal)?.let { - // add new artist, switch all old references, then delete old one - database.insert(it) - swapArtists(element, it, database) - } - } catch (e: Exception) { - // don't touch anything if ytm fails --> keep old artist - } - } else { - // swap with database artist - swapArtists(element, databaseArtistMatch.first().artist, database) - } - } - ) - } - } -} - - -/** - * Swap all participation(s) with old artist to use new artist - * - * p.s. This is here instead of DatabaseDao because it won't compile there because - * "oooga boooga error in generated code" - */ -suspend fun swapArtists(old: ArtistEntity, new: ArtistEntity, database: MusicDatabase) { - if (database.artist(old.id).first() == null) { - throw Exception("Attempting to swap with non-existent old artist in database with id: ${old.id}") - } - if (database.artist(new.id).first() == null) { - throw Exception("Attempting to swap with non-existent new artist in database with id: ${new.id}") - } - - // update participation(s) - database.updateSongArtistMap(old.id, new.id) - database.updateAlbumArtistMap(old.id, new.id) - - // nuke old artist - database.delete(old) -} - -/** - * Remove inaccessible songs from the library - */ -private fun disableSongs(database: MusicDatabase) { - runBlocking(Dispatchers.IO) { - // get list of all songs in db - val allSongs = database.allLocalSongs().first() - - for (song in allSongs) { - if (song.song.localPath == null) { - database.inLibrary(song.id, null) - continue - } - - val f = File(song.song.localPath) - // we can't play non-existent file or if it becomes a directory - if (!f.exists() || f.isDirectory()) { - if (SCANNER_DEBUG) - Timber.tag(TAG).d("Disabling song ${song.song.localPath}") - database.inLibrary(song.song.id, null) - } - } - } -} - - -/** - * Destroys all local library data (local songs and artists, does not include YTM downloads) - * from the database - */ -fun nukeLocalDB(database: MusicDatabase) { - Timber.tag(TAG).w("NUKING LOCAL FILE LIBRARY FROM DATABASE! Nuke status: ${database.nukeLocalData()}") -} - -/** - * Destroys all local library data (local songs and artists, does not include YTM downloads) - * from the database, then rebuilds it. - * - * @param database - * @param newSongs - * @param matchCriteria How lax should the scanner be - * @param strictFileNames Whether to consider file names - */ -fun destructiveRescanDB( - database: MusicDatabase, - newSongs: List, - matchCriteria: ScannerMatchCriteria, - strictFileNames: Boolean -) { - nukeLocalDB(database) - syncDB(database, newSongs, matchCriteria, strictFileNames) -} - -/** - * Check if artists are the same - * - * Both null == same artists - * Either null == different artists - */ -fun compareArtist(a: List, b: List): Boolean { - if (a.isEmpty() && b.isEmpty()) { - return true - } else if (a.isEmpty() || b.isEmpty()) { - return false - } - - // compare entries - if (a.size != b.size) { - return false - } - val matchingArtists = a.filter { artist -> - b.any { it.name.lowercase(Locale.getDefault()) == artist.name.lowercase(Locale.getDefault()) } - } - - return matchingArtists.size == a.size -} - -/** - * Check the similarity of a song - * - * @param a - * @param b - * @param matchStrength How lax should the scanner be - * @param strictFileNames Whether to consider file names - */ -fun compareSong(a: Song, b: Song, matchStrength: ScannerMatchCriteria, strictFileNames: Boolean): Boolean { - // if match file names - if (strictFileNames && - (a.song.localPath?.substringAfterLast('/') != - b.song.localPath?.substringAfterLast('/')) - ) { - return false - } - - /** - * Compare file paths - * - * I draw the "user error" line here - */ - fun closeEnough(): Boolean { - return a.song.localPath == b.song.localPath - } - - // compare songs based on scanner strength - return when (matchStrength) { - ScannerMatchCriteria.LEVEL_1 -> a.song.title == b.song.title - ScannerMatchCriteria.LEVEL_2 -> closeEnough() || (a.song.title == b.song.title && - compareArtist(a.artists, b.artists)) - - ScannerMatchCriteria.LEVEL_3 -> closeEnough() || (a.song.title == b.song.title && - compareArtist(a.artists, b.artists) /* && album compare goes here */ ) - } -} - /** * ========================== * Various misc helpers @@ -1084,11 +106,18 @@ fun getLocalThumbnail(path: String?, resize: Boolean): Bitmap? { /** - * Get cached directory tree + * Get cached DirectoryTree */ fun getDirectoryTree(): DirectoryTree? { if (cachedDirectoryTree == null) { return null } return cachedDirectoryTree +} + +/** + * Cache a DirectoryTree + */ +fun cacheDirectoryTree(new: DirectoryTree?) { + cachedDirectoryTree = new } \ No newline at end of file diff --git a/app/src/main/java/com/dd3boh/outertune/utils/scanners/LocalMediaScanner.kt b/app/src/main/java/com/dd3boh/outertune/utils/scanners/LocalMediaScanner.kt new file mode 100644 index 000000000..0b03e09ad --- /dev/null +++ b/app/src/main/java/com/dd3boh/outertune/utils/scanners/LocalMediaScanner.kt @@ -0,0 +1,857 @@ +package com.dd3boh.outertune.utils.scanners + +import android.content.ContentResolver +import android.content.Context +import android.media.MediaMetadataRetriever +import android.os.Build +import android.provider.MediaStore +import com.dd3boh.outertune.constants.ScannerImpl +import com.dd3boh.outertune.constants.ScannerMatchCriteria +import com.dd3boh.outertune.db.MusicDatabase +import com.dd3boh.outertune.db.entities.AlbumEntity +import com.dd3boh.outertune.db.entities.ArtistEntity +import com.dd3boh.outertune.db.entities.FormatEntity +import com.dd3boh.outertune.db.entities.Song +import com.dd3boh.outertune.db.entities.SongArtistMap +import com.dd3boh.outertune.db.entities.SongEntity +import com.dd3boh.outertune.models.DirectoryTree +import com.dd3boh.outertune.models.toMediaMetadata +import com.dd3boh.outertune.ui.utils.ARTIST_SEPARATORS +import com.dd3boh.outertune.ui.utils.SCANNER_CRASH_AT_FIRST_ERROR +import com.dd3boh.outertune.ui.utils.SCANNER_DEBUG +import com.dd3boh.outertune.ui.utils.SYNC_SCANNER +import com.dd3boh.outertune.models.SongTempData +import com.dd3boh.outertune.ui.utils.cacheDirectoryTree +import com.dd3boh.outertune.ui.utils.projection +import com.dd3boh.outertune.ui.utils.scannerSession +import com.dd3boh.outertune.ui.utils.sdcardRoot +import com.dd3boh.outertune.ui.utils.testScanPaths +import com.zionhuang.innertube.YouTube +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.firstOrNull +import kotlinx.coroutines.runBlocking +import timber.log.Timber +import java.io.File +import java.time.LocalDateTime +import java.util.Locale + +class LocalMediaScanner { + + /** + * Compiles a song with all it's necessary metadata. Unlike MediaStore, + * this also supports multiple artists, multiple genres (TBD), and a few extra details (TBD). + */ + private fun advancedScan( + basicData: SongTempData, + database: MusicDatabase, + scannerImpl: ScannerImpl, + ): Song { + val artists = ArrayList() +// var generes +// var year: String? = null + + // MediaStore mode + var rawArtists = basicData.artist + + try { + // decide which scanner to use + val scanner = getAdvancedScanner(scannerImpl) + var ffmpegData: ExtraMetadataWrapper? = null + if (scannerImpl == ScannerImpl.MEDIASTORE_FFPROBE) { + ffmpegData = scanner?.getMediaStoreSupplement(basicData.path) + rawArtists = ffmpegData?.artists + } else if (scannerImpl == ScannerImpl.FFPROBE) { + ffmpegData = scanner?.getAllMetadata(basicData.path, basicData.formatEntity) + rawArtists = ffmpegData?.artists + } + + // parse data + rawArtists?.split(ARTIST_SEPARATORS)?.forEach { element -> + val artistVal = element.trim() + artists.add(ArtistEntity("LA${ArtistEntity.generateArtistId()}", artistVal, isLocal = true)) + } + + // file format info + if (scannerImpl == ScannerImpl.FFPROBE && ffmpegData?.format != null) { + database.query { + upsert( + ffmpegData.format!! + ) + } + } else { // MEDIASTORE_FFPROBE and MEDIASTORE + database.query { + upsert( + basicData.formatEntity + ) + } + } + } catch (e: Exception) { + if (SCANNER_CRASH_AT_FIRST_ERROR) { + throw Exception("HALTING AT FIRST SCANNER ERROR " + e.message) // debug + } + // fallback on media store + if (SCANNER_DEBUG) { + Timber.tag(TAG).d( + "ERROR READING ARTIST METADATA: ${e.message}" + + " Falling back on MediaStore for ${basicData.path}" + ) + e.printStackTrace() + } + + if (basicData.artist?.isNotBlank() == true) { + if (SCANNER_DEBUG) { + Timber.tag(TAG).d( + "Adding local artist with name: ${basicData.artist}" + ) + artists.add(ArtistEntity(ArtistEntity.generateArtistId(), basicData.artist, isLocal = true)) + } + } + } + + return Song( + SongEntity( + basicData.id, + basicData.title, + (basicData.duration / 1000), // we use seconds for duration + albumId = basicData.albumID, + albumName = basicData.album, + isLocal = true, + inLibrary = LocalDateTime.now(), + localPath = basicData.path + ), + artists, + // album not working + basicData.albumID?.let { + basicData.album?.let { it1 -> + AlbumEntity( + it, + title = it1, + duration = 0, + songCount = 1 + ) + } + } + ) + } + + /** + * Dev uses + */ + fun scanLocal(context: Context, database: MusicDatabase, scannerImpl: ScannerImpl) = + scanLocal(context, database, testScanPaths, scannerImpl) + + /** + * Scan MediaStore for songs given a list of paths to scan for. + * This will replace all data in the database for a given song. + * + * @param context Context + * @param scanPaths List of whitelist paths to scan under. This assumes + * the current directory is /storage/emulated/0/ a.k.a, /sdcard. + * For example, to scan under Music and Documents/songs --> ("Music", Documents/songs) + */ + @OptIn(ExperimentalCoroutinesApi::class) + fun scanLocal( + context: Context, + database: MusicDatabase, + scanPaths: ArrayList, + scannerImpl: ScannerImpl, + ): MutableStateFlow { + val newDirectoryStructure = DirectoryTree(sdcardRoot) + val contentResolver: ContentResolver = context.contentResolver + + // Query for audio files + val cursor = contentResolver.query( + MediaStore.Audio.Media.EXTERNAL_CONTENT_URI, + projection, + "${MediaStore.Audio.Media.IS_MUSIC} != 0 AND ${MediaStore.Audio.Media.DATA} LIKE ?", + scanPaths.map { "$sdcardRoot$it%" }.toTypedArray(), // whitelist paths + null + ) + Timber.tag(TAG).d("------------ SCAN: Starting Full Scanner ------------") + + val scannerJobs = ArrayList>() + runBlocking { + // MediaStore is our "basic" scanner & file discovery + cursor?.use { cursor -> + // Columns indices + val nameColumn = cursor.getColumnIndexOrThrow(MediaStore.Audio.Media.DISPLAY_NAME) + val titleColumn = cursor.getColumnIndexOrThrow(MediaStore.Audio.Media.TITLE) + val durationColumn = cursor.getColumnIndexOrThrow(MediaStore.Audio.Media.DURATION) + val artistIdColumn = cursor.getColumnIndexOrThrow(MediaStore.Audio.Media.ALBUM_ID) + val artistColumn = cursor.getColumnIndexOrThrow(MediaStore.Audio.Media.ARTIST) + val albumIDColumn = cursor.getColumnIndexOrThrow(MediaStore.Audio.Media.ALBUM_ID) + val albumColumn = cursor.getColumnIndexOrThrow(MediaStore.Audio.Media.ALBUM) + val pathColumn = cursor.getColumnIndexOrThrow(MediaStore.Audio.Media.RELATIVE_PATH) + + val mimeColumn = cursor.getColumnIndexOrThrow(MediaStore.Audio.Media.MIME_TYPE) + val bitrateColumn = cursor.getColumnIndexOrThrow(MediaStore.Audio.Media.BITRATE) + + while (cursor.moveToNext()) { + val id = SongEntity.generateSongId() + val name = cursor.getString(nameColumn) // file name + val title = cursor.getString(titleColumn) // song title + val duration = cursor.getInt(durationColumn) + val artist = cursor.getString(artistColumn) + val artistID = cursor.getString(artistIdColumn) + val albumID = cursor.getString(albumIDColumn) + val album = cursor.getString(albumColumn) + val path = cursor.getString(pathColumn) + + // extra stream info + val bitrate = cursor.getInt(bitrateColumn) + val mime = cursor.getString(mimeColumn) + + if (SCANNER_DEBUG) + Timber.tag(TAG) + .d("ID: $id, Name: $name, ARTIST: $artist, PATH: $path") + + // append song to list + // media store doesn't support multi artists... + // do not link album (and whatever song id) with youtube yet, figure that out later + + if (!SYNC_SCANNER) { + // use async scanner + scannerJobs.add( + async(scannerSession) { + advancedScan( + SongTempData( + id, "$sdcardRoot$path$name", title, duration, + artist, artistID, album, albumID, + FormatEntity( + id = id, + itag = -1, + mimeType = mime, + codecs = mime.substringAfter('/'), + bitrate = bitrate, + sampleRate = -1, + contentLength = duration.toLong(), + loudnessDb = null, + playbackUrl = null + ), + ), + database, scannerImpl, + ) + } + ) + } else { + // force synchronous scanning of songs + val toInsert = advancedScan( + SongTempData( + id, "$sdcardRoot$path$name", title, duration, + artist, artistID, album, albumID, + FormatEntity( + id = id, + itag = -1, + mimeType = mime, + codecs = mime.substringAfter('/'), + bitrate = bitrate, + sampleRate = -1, + contentLength = duration.toLong(), + loudnessDb = null, + playbackUrl = null + ) + ), database, scannerImpl + ) + toInsert.song.localPath?.let { s -> + newDirectoryStructure.insert( + s.substringAfter(sdcardRoot), toInsert + ) + } + } + } + } + + if (!SYNC_SCANNER) { + // use async scanner + scannerJobs.awaitAll() + } + } + + // build the tree + scannerJobs.forEach { + val song = it.getCompleted() + + song.song.localPath?.let { s -> + newDirectoryStructure.insert( + s.substringAfter(sdcardRoot), song + ) + } + } + + Timber.tag(TAG).d("------------ SCAN: Finished Full Scanner ------------") + cacheDirectoryTree(newDirectoryStructure) + return MutableStateFlow(newDirectoryStructure) + } + + /** + * Update the Database with local files + * + * @param database + * @param newSongs + * @param matchStrength How lax should the scanner be + * @param strictFileNames Whether to consider file names + * @param refreshExisting Setting this this to true will updated existing songs + * with new information, else existing song's data will not be touched, regardless + * whether it was actually changed on disk + * + * Inserts a song if not found + * Updates a song information depending on if refreshExisting value + */ + fun syncDB( + database: MusicDatabase, + newSongs: List, + matchStrength: ScannerMatchCriteria, + strictFileNames: Boolean, + refreshExisting: Boolean? = false + ) { + Timber.tag(TAG).d("------------ SYNC: Starting Local Library Sync ------------") + Timber.tag(TAG).d("Entries to process: ${newSongs.size}") + + newSongs.forEach { song -> + val querySong = database.searchSongsInclNotInLibrary(song.song.title) + + runBlocking(Dispatchers.IO) { + + // check if this song is known to the library + val songMatch = querySong.first().filter { + return@filter compareSong(it, song, matchStrength, strictFileNames) + } + + if (SCANNER_DEBUG) { + Timber.tag(TAG) + .d("Found songs that match: ${songMatch.size}, Total results from database: ${querySong.first().size}") + if (songMatch.isNotEmpty()) { + Timber.tag(TAG) + .d("FIRST Found songs ${songMatch.first().song.title}") + } + } + + + if (songMatch.isNotEmpty() && refreshExisting == true) { // known song, update the song info in the database + Timber.tag(TAG) + .d("Found in database, updating song: ${song.song.title}") + val songToUpdate = songMatch.first() + database.update(songToUpdate.song) + + // destroy existing artist links + database.unlinkSongArtists(songToUpdate.id) + + // update artists + var artistPos = 0 + song.artists.forEach { + val dbArtist = database.searchArtists(it.name).firstOrNull()?.firstOrNull() + + if (dbArtist == null) { + // artist does not exist in db, add it then link it + database.insert(it) + database.insert(SongArtistMap(songToUpdate.id, it.id, artistPos)) + } else { + // artist does exist in db, link to it + database.insert(SongArtistMap(songToUpdate.id, dbArtist.artist.id, artistPos)) + } + + artistPos++ + } + } else if (songMatch.isEmpty()) { // new song + if (SCANNER_DEBUG) + Timber.tag(TAG) + .d("NOT found in database, adding song: ${song.song.title}") + database.insert(song.toMediaMetadata()) + } + // do not delete songs from database automatically, we just disable them + disableSongs(database) + } + } + Timber.tag(TAG).d("------------ SYNC: Finished Local Library Sync ------------") + } + + /** + * A faster scanner implementation that adds new songs to the database, + * and does not touch older songs entires (apart from removing + * inacessable songs from libaray). + * + * No remote artist lookup is done + * + * WARNING: cachedDirectoryTree is not refreshed and may lead to inconsistencies. + * It is highly recommend to rebuild the tree after scanner operation + * + * @param newSongs List of songs. This is expecting a barebones DirectoryTree + * (only paths are necessary), thus you may use the output of refreshLocal().toList() + */ + @OptIn(ExperimentalCoroutinesApi::class) + fun quickSync( + database: MusicDatabase, + newSongs: List, + matchCriteria: ScannerMatchCriteria, + strictFileNames: Boolean, + scannerImpl: ScannerImpl, + ) { + Timber.tag(TAG).d("------------ SYNC: Starting Quick (additive delta) Library Sync ------------") + Timber.tag(TAG).d("Entries to process: ${newSongs.size}") + + val mData = MediaMetadataRetriever() + + runBlocking(Dispatchers.IO) { + // get list of all songs in db, then get songs unknown to the database + val allSongs = database.allLocalSongs().first() + val delta = newSongs.filterNot { + allSongs.any { dbSong -> compareSong(it, dbSong, matchCriteria, strictFileNames) } + } + + val artistsWithMetadata = ArrayList() + val scannerJobs = ArrayList>() + runBlocking { + // Get song basic metadata + delta.forEach { s -> + mData.setDataSource(s.song.localPath) + + val id = SongEntity.generateSongId() + val title = + mData.extractMetadata(MediaMetadataRetriever.METADATA_KEY_TITLE).let { it ?: "" } // song title + val duration = + Integer.parseInt(mData.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION)!!) + val artist = mData.extractMetadata(MediaMetadataRetriever.METADATA_KEY_ARTIST) + val artistID = if (artist == null) ArtistEntity.generateArtistId() else null + val album = mData.extractMetadata(MediaMetadataRetriever.METADATA_KEY_ALBUM) + val albumID = if (album == null) AlbumEntity.generateAlbumId() else null + // path should never be null since its coming from directory tree scanner + // but Kotlin is too dumb to care. Just ruthlessly suppress the error... + val path = "" + s.song.localPath + + // extra stream info + val bitrate = mData.extractMetadata(MediaMetadataRetriever.METADATA_KEY_BITRATE)?.let { + Integer.parseInt( + it + ) + } + val mime = "" + mData.extractMetadata(MediaMetadataRetriever.METADATA_KEY_MIMETYPE) + var sampleRate = -1 + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + sampleRate = mData.extractMetadata(MediaMetadataRetriever.METADATA_KEY_SAMPLERATE)?.let { + Integer.parseInt( + it + ) + }!! + } + + + if (SCANNER_DEBUG) + Timber.tag(TAG) + .d("ID: $id, Title: $title, ARTIST: $artist, PATH: $path") + + // append song to list + // media store doesn't support multi artists... + // do not link album (and whatever song id) with youtube yet, figure that out later + + if (!SYNC_SCANNER) { + // use async scanner + scannerJobs.add( + async(scannerSession) { + advancedScan( + SongTempData( + id, path, title, duration, artist, artistID, album, albumID, + FormatEntity( + id = id, + itag = -1, + mimeType = mime, + codecs = mime.substringAfter('/'), + bitrate = bitrate ?: -1, + sampleRate = sampleRate, + contentLength = duration.toLong(), + loudnessDb = null, + playbackUrl = null + ) + ), database, scannerImpl // no online artist lookup + ) + } + ) + } else { + // force synchronous scanning of songs + val toInsert = advancedScan( + SongTempData( + id, path, title, duration, artist, artistID, album, albumID, + FormatEntity( + id = id, + itag = -1, + mimeType = mime, + codecs = mime.substringAfter('/'), + bitrate = bitrate ?: -1, + sampleRate = sampleRate, + contentLength = duration.toLong(), + loudnessDb = null, + playbackUrl = null + ) + ), database, scannerImpl + ) + artistsWithMetadata.add(toInsert) + } + } + } + + if (!SYNC_SCANNER) { + // use async scanner + scannerJobs.awaitAll() + } + + // add to finished list + scannerJobs.forEach { + artistsWithMetadata.add(it.getCompleted()) + } + + if (delta.isNotEmpty()) { + syncDB(database, artistsWithMetadata, matchCriteria, strictFileNames) + } + + disableSongs(database) + } + Timber.tag(TAG).d("------------ SYNC: Finished Quick (additive delta) Library Sync ------------") + } + + /** + * Converts all local artists to remote artists if possible + */ + fun localToRemoteArtist(database: MusicDatabase) { + runBlocking(Dispatchers.IO) { + val allLocal = database.allLocalArtists().first() + val scannerJobs = ArrayList>() + + allLocal.forEach { element -> + val artistVal = element.name.trim() + + // check if this artist exists in DB already + val databaseArtistMatch = + runBlocking(Dispatchers.IO) { + database.searchArtists(artistVal).first().filter { artist -> + // only look for remote artists here + return@filter artist.artist.name == artistVal && !artist.artist.isLocalArtist + } + } + + if (SCANNER_DEBUG) + Timber.tag(TAG) + .d("ARTIST FOUND IN DB??? Results size: ${databaseArtistMatch.size}") + + scannerJobs.add( + async(scannerSession) { + // resolve artist from YTM if not found in DB + if (databaseArtistMatch.isEmpty()) { + try { + youtubeArtistLookup(artistVal)?.let { + // add new artist, switch all old references, then delete old one + database.insert(it) + swapArtists(element, it, database) + } + } catch (e: Exception) { + // don't touch anything if ytm fails --> keep old artist + } + } else { + // swap with database artist + swapArtists(element, databaseArtistMatch.first().artist, database) + } + } + ) + } + } + } + + + /** + * Remove inaccessible songs from the library + */ + private fun disableSongs(database: MusicDatabase) { + runBlocking(Dispatchers.IO) { + // get list of all songs in db + val allSongs = database.allLocalSongs().first() + + for (song in allSongs) { + if (song.song.localPath == null) { + database.inLibrary(song.id, null) + continue + } + + val f = File(song.song.localPath) + // we can't play non-existent file or if it becomes a directory + if (!f.exists() || f.isDirectory()) { + if (SCANNER_DEBUG) + Timber.tag(TAG).d("Disabling song ${song.song.localPath}") + database.inLibrary(song.song.id, null) + } + } + } + } + + + /** + * Destroys all local library data (local songs and artists, does not include YTM downloads) + * from the database + */ + fun nukeLocalDB(database: MusicDatabase) { + Timber.tag(TAG) + .w("NUKING LOCAL FILE LIBRARY FROM DATABASE! Nuke status: ${database.nukeLocalData()}") + } + + /** + * Destroys all local library data (local songs and artists, does not include YTM downloads) + * from the database, then rebuilds it. + * + * @param database + * @param newSongs + * @param matchCriteria How lax should the scanner be + * @param strictFileNames Whether to consider file names + */ + fun destructiveRescanDB( + database: MusicDatabase, + newSongs: List, + matchCriteria: ScannerMatchCriteria, + strictFileNames: Boolean + ) { + nukeLocalDB(database) + syncDB(database, newSongs, matchCriteria, strictFileNames) + } + + + companion object { + // do not put any thing that should adhere to the scanner lock in here + const val TAG = "LocalMediaScanner" + + private var localScanner: LocalMediaScanner? = null + private var advancedScannerImpl: MetadataScanner? = null + + /** + * ========================== + * Scanner management + * ========================== + */ + + /** + * Trust me bro, it should never be null + */ + fun getScanner(): LocalMediaScanner { + if (localScanner == null) { + localScanner = LocalMediaScanner() + } + + return localScanner!! + } + + fun destroyScanner() { + localScanner = null + unloadAdvancedScanner() + } + + /** + * TODO: Docs here + */ + fun getAdvancedScanner(scannerImpl: ScannerImpl): MetadataScanner? { + // kotlin won't let me return MetadataScanner even if it cant possibly be null broooo + return when (scannerImpl) { + ScannerImpl.FFPROBE, ScannerImpl.MEDIASTORE_FFPROBE -> + if (advancedScannerImpl is FFProbeScanner) advancedScannerImpl else FFProbeScanner() + + ScannerImpl.MEDIASTORE -> null + } + } + + fun unloadAdvancedScanner() { + advancedScannerImpl = null + } + + + /** + * ========================== + * Scanner helpers + * ========================== + */ + + + /** + * Dev uses + */ + fun refreshLocal(context: Context, database: MusicDatabase) = + refreshLocal(context, database, testScanPaths) + + /** + * Quickly rebuild a skeleton directory tree of local files based on the database + * + * Notes: + * If files move around, that's on you to re run the scanner. + * If the metadata changes, that's also on you to re run the scanner. + * + * @param context Context + * @param scanPaths List of whitelist paths to scan under. This assumes + * the current directory is /storage/emulated/0/ a.k.a, /sdcard. + * For example, to scan under Music and Documents/songs --> ("Music", Documents/songs) + */ + fun refreshLocal( + context: Context, + database: MusicDatabase, + scanPaths: ArrayList + ): MutableStateFlow { + val newDirectoryStructure = DirectoryTree(sdcardRoot) + + // get songs from db + var existingSongs: List + runBlocking(Dispatchers.IO) { + existingSongs = database.allLocalSongs().first() + } + + // Query for audio files + val contentResolver: ContentResolver = context.contentResolver + val cursor = contentResolver.query( + MediaStore.Audio.Media.EXTERNAL_CONTENT_URI, + projection, + "${MediaStore.Audio.Media.IS_MUSIC} != 0 AND ${MediaStore.Audio.Media.DATA} LIKE ?", + scanPaths.map { "$sdcardRoot$it%" }.toTypedArray(), // whitelist paths + null + ) + Timber.tag(TAG).d("------------ SCAN: Starting Quick Directory Rebuild ------------") + cursor?.use { localCursor -> + // Columns indices + val nameColumn = localCursor.getColumnIndexOrThrow(MediaStore.Audio.Media.DISPLAY_NAME) + val pathColumn = localCursor.getColumnIndexOrThrow(MediaStore.Audio.Media.RELATIVE_PATH) + + while (localCursor.moveToNext()) { + val name = localCursor.getString(nameColumn) // file name + val path = localCursor.getString(pathColumn) + + if (SCANNER_DEBUG) + Timber.tag(TAG).d("Quick scanner: PATH: $path") + + // Build directory tree with existing files + val possibleMatch = existingSongs.firstOrNull() { it.song.localPath == "$sdcardRoot$path$name" } + + if (possibleMatch != null) { + newDirectoryStructure.insert("$path$name", possibleMatch) + } + + } + } + + Timber.tag(TAG).d("------------ SCAN: Finished Quick Directory Rebuild ------------") + cacheDirectoryTree(newDirectoryStructure) + return MutableStateFlow(newDirectoryStructure) + } + + /** + * Check if artists are the same + * + * Both null == same artists + * Either null == different artists + */ + fun compareArtist(a: List, b: List): Boolean { + if (a.isEmpty() && b.isEmpty()) { + return true + } else if (a.isEmpty() || b.isEmpty()) { + return false + } + + // compare entries + if (a.size != b.size) { + return false + } + val matchingArtists = a.filter { artist -> + b.any { it.name.lowercase(Locale.getDefault()) == artist.name.lowercase(Locale.getDefault()) } + } + + return matchingArtists.size == a.size + } + + /** + * Check the similarity of a song + * + * @param a + * @param b + * @param matchStrength How lax should the scanner be + * @param strictFileNames Whether to consider file names + */ + fun compareSong(a: Song, b: Song, matchStrength: ScannerMatchCriteria, strictFileNames: Boolean): Boolean { + // if match file names + if (strictFileNames && + (a.song.localPath?.substringAfterLast('/') != + b.song.localPath?.substringAfterLast('/')) + ) { + return false + } + + /** + * Compare file paths + * + * I draw the "user error" line here + */ + fun closeEnough(): Boolean { + return a.song.localPath == b.song.localPath + } + + // compare songs based on scanner strength + return when (matchStrength) { + ScannerMatchCriteria.LEVEL_1 -> a.song.title == b.song.title + ScannerMatchCriteria.LEVEL_2 -> closeEnough() || (a.song.title == b.song.title && + compareArtist(a.artists, b.artists)) + + ScannerMatchCriteria.LEVEL_3 -> closeEnough() || (a.song.title == b.song.title && + compareArtist(a.artists, b.artists) /* && album compare goes here */) + } + } + + /** + * Search for an artist on YouTube Music. + * + * If no artist is found, create one locally + */ + fun youtubeArtistLookup(query: String): ArtistEntity? { + var ytmResult: ArtistEntity? = null + + // hit up YouTube for artist + runBlocking(Dispatchers.IO) { + YouTube.search(query, YouTube.SearchFilter.FILTER_ARTIST).onSuccess { result -> + + val foundArtist = result.items.firstOrNull { + it.title.lowercase(Locale.getDefault()) == query.lowercase(Locale.getDefault()) + } ?: throw Exception("Failed to search: Artist not found on YouTube Music") + ytmResult = ArtistEntity( + foundArtist.id, + foundArtist.title, + foundArtist.thumbnail + ) + + if (SCANNER_DEBUG) + Timber.tag(TAG) + .d("Found remote artist: ${result.items.first().title}") + }.onFailure { + throw Exception("Failed to search on YouTube Music") + } + + } + + return ytmResult + } + + /** + * Swap all participation(s) with old artist to use new artist + * + * p.s. This is here instead of DatabaseDao because it won't compile there because + * "oooga boooga error in generated code" + */ + suspend fun swapArtists(old: ArtistEntity, new: ArtistEntity, database: MusicDatabase) { + if (database.artist(old.id).first() == null) { + throw Exception("Attempting to swap with non-existent old artist in database with id: ${old.id}") + } + if (database.artist(new.id).first() == null) { + throw Exception("Attempting to swap with non-existent new artist in database with id: ${new.id}") + } + + // update participation(s) + database.updateSongArtistMap(old.id, new.id) + database.updateAlbumArtistMap(old.id, new.id) + + // nuke old artist + database.delete(old) + } + } +} diff --git a/app/src/main/java/com/dd3boh/outertune/viewmodels/LibraryViewModels.kt b/app/src/main/java/com/dd3boh/outertune/viewmodels/LibraryViewModels.kt index de0957e0c..a835ec4d5 100644 --- a/app/src/main/java/com/dd3boh/outertune/viewmodels/LibraryViewModels.kt +++ b/app/src/main/java/com/dd3boh/outertune/viewmodels/LibraryViewModels.kt @@ -18,11 +18,11 @@ import com.dd3boh.outertune.db.entities.Song import com.dd3boh.outertune.extensions.reversed import com.dd3boh.outertune.extensions.toEnum import com.dd3boh.outertune.playback.DownloadUtil -import com.dd3boh.outertune.ui.utils.DirectoryTree -import com.dd3boh.outertune.ui.utils.refreshLocal +import com.dd3boh.outertune.models.DirectoryTree import com.dd3boh.outertune.utils.SyncUtils import com.dd3boh.outertune.utils.dataStore import com.dd3boh.outertune.utils.reportException +import com.dd3boh.outertune.utils.scanners.LocalMediaScanner.Companion.refreshLocal import dagger.hilt.android.lifecycle.HiltViewModel import dagger.hilt.android.qualifiers.ApplicationContext import kotlinx.coroutines.Dispatchers