From dbd03de0caa21165f1720dd7466fda577bc41064 Mon Sep 17 00:00:00 2001 From: datawhores Date: Sun, 26 May 2024 15:30:17 -0500 Subject: [PATCH] add some improvements for discord log and final log --- ofscraper/commands/check.py | 2 +- ofscraper/commands/helpers/context.py | 8 ++++-- ofscraper/commands/scraper/actions/like.py | 17 ++++++------ ofscraper/download/download.py | 11 ++++---- ofscraper/download/downloadbatch.py | 10 ++----- ofscraper/download/downloadnormal.py | 11 +++----- ofscraper/download/shared/utils/log.py | 31 ++++++++++++++-------- ofscraper/utils/console.py | 4 +-- ofscraper/utils/logs/classes.py | 17 +++++++++--- ofscraper/utils/logs/other.py | 2 +- ofscraper/utils/logs/stdout.py | 11 ++++---- 11 files changed, 69 insertions(+), 55 deletions(-) diff --git a/ofscraper/commands/check.py b/ofscraper/commands/check.py index e40ef3ecf..c1a4ea29c 100644 --- a/ofscraper/commands/check.py +++ b/ofscraper/commands/check.py @@ -70,7 +70,7 @@ def process_item(): try: row, key = table.row_queue.get() except Exception as E: - log.error(f"Error getting item from queue: {E}") + log.debug(f"Error getting item from queue: {E}") return for count, _ in enumerate(range(0, 2)): try: diff --git a/ofscraper/commands/helpers/context.py b/ofscraper/commands/helpers/context.py index 148e68bdf..553311d0f 100644 --- a/ofscraper/commands/helpers/context.py +++ b/ofscraper/commands/helpers/context.py @@ -1,6 +1,8 @@ -import contextlib import logging import traceback +from itertools import chain + + import ofscraper.models.selector as selector import ofscraper.utils.args.helpers.areas as areas @@ -45,7 +47,9 @@ async def wrapper(userdata,session,*args, **kwargs): raise e finally: progress_utils.increment_user_activity() - log.warning(data) + for record in chain.from_iterable(data): + log.warning(record) + return wrapper diff --git a/ofscraper/commands/scraper/actions/like.py b/ofscraper/commands/scraper/actions/like.py index a86bd2dce..c34d8f3fe 100644 --- a/ofscraper/commands/scraper/actions/like.py +++ b/ofscraper/commands/scraper/actions/like.py @@ -21,7 +21,8 @@ import ofscraper.utils.constants as constants import ofscraper.utils.context.exit as exit import ofscraper.utils.live.screens as progress_utils -import ofscraper.api.profile as profile +from rich.markup import escape +from rich.text import Text log = logging.getLogger("shared") @@ -133,17 +134,17 @@ def _like(model_id, username,ids: list, like_action: bool): else: sleep_duration = stable_sleep_duration if out == 1: - progress_utils.increment_like_Ftask(task2) + progress_utils.increment_like_task(task2) progress_utils.increment_like_task(task) time.sleep(sleep_duration) - out="[{username}] {post} post checked, {liked} post changes to {action}".format(post=progress_utils.get_like_task(task2).completed,liked=progress_utils.get_like_task(task).completed, - username = username, - action="liked" if like_action else "unliked" - ) + title="Liked" if like_action else "Unliked" + liked=progress_utils.get_like_task(task2).completed + post=progress_utils.get_like_task(task).completed + action=title.lower() + text_out=f"[bold]\\[{username}][/bold] [bold]\\[Action {title}][/bold] ({post} post checked, {liked} post changes to {action})" progress_utils.remove_like_task(task) progress_utils.remove_like_task(task2) - return out - + return text_out def _toggle_like_requests(c, id, model_id): diff --git a/ofscraper/download/download.py b/ofscraper/download/download.py index 388a7ab28..2a4168721 100644 --- a/ofscraper/download/download.py +++ b/ofscraper/download/download.py @@ -13,6 +13,7 @@ import ofscraper.utils.system.system as system from ofscraper.download.shared.utils.text import textDownloader from ofscraper.utils.context.run_async import run +from ofscraper.download.shared.utils.log import empty_log @run @@ -26,10 +27,11 @@ async def download_process(username, model_id, medialist, posts=None): async def download_picker(username, model_id, medialist): if len(medialist) == 0: + out=empty_log(username) logging.getLogger("shared").error( - f"[bold]{username}[/bold] ({0} photos, {0} videos, {0} audios, {0} skipped, {0} failed)" + out ) - return 0, 0, 0, 0, 0 + return out elif ( system.getcpu_count() > 1 and ( @@ -71,7 +73,4 @@ def download_post_process(username, model_id, medialist, postlist): log.traceback_(traceback.format_exc()) -def empty_log(username): - logging.getLogger("shared").error( - f"[bold]{username}[/bold] ({0} photos, {0} videos, {0} audios, {0} skipped, {0} failed)" - ) + diff --git a/ofscraper/download/downloadbatch.py b/ofscraper/download/downloadbatch.py index a16392344..c5fdeab81 100644 --- a/ofscraper/download/downloadbatch.py +++ b/ofscraper/download/downloadbatch.py @@ -36,7 +36,7 @@ get_medialog, subProcessVariableInit, ) -from ofscraper.download.shared.utils.log import final_log, log_download_progress +from ofscraper.download.shared.utils.log import final_log, final_log_text ,log_download_progress from ofscraper.download.shared.utils.metadata import metadata from ofscraper.download.shared.utils.paths import addGlobalDir, setDirectoriesDate from ofscraper.download.shared.utils.progress import convert_num_bytes @@ -199,13 +199,7 @@ def process_dicts(username, model_id, filtered_medialist): with exit.DelayedKeyboardInterrupt(): raise E final_log(username) - return ( - common_globals.photo_count, - common_globals.video_count, - common_globals.audio_count, - common_globals.forced_skipped, - common_globals.skipped, - ) + return final_log_text(username) def queue_process(pipe_, task1, total): diff --git a/ofscraper/download/downloadnormal.py b/ofscraper/download/downloadnormal.py index 6788430c7..91b22e01e 100644 --- a/ofscraper/download/downloadnormal.py +++ b/ofscraper/download/downloadnormal.py @@ -29,7 +29,7 @@ from ofscraper.download.main_download import main_download from ofscraper.download.shared.classes.session import download_session from ofscraper.download.shared.common.general import get_medialog -from ofscraper.download.shared.utils.log import final_log, log_download_progress +from ofscraper.download.shared.utils.log import final_log, final_log_text,log_download_progress from ofscraper.download.shared.utils.metadata import metadata from ofscraper.download.shared.utils.paths import setDirectoriesDate from ofscraper.download.shared.utils.progress import convert_num_bytes @@ -148,13 +148,8 @@ async def process_dicts(username, model_id, medialist): log_thread.join() other_thread.join() if other_thread else None final_log(username, log=logging.getLogger("shared")) - return ( - common_globals.photo_count, - common_globals.video_count, - common_globals.audio_count, - common_globals.forced_skipped, - common_globals.skipped, - ) + return final_log_text(username) + except Exception as E: with exit.DelayedKeyboardInterrupt(): diff --git a/ofscraper/download/shared/utils/log.py b/ofscraper/download/shared/utils/log.py index 76196fdae..736265a90 100644 --- a/ofscraper/download/shared/utils/log.py +++ b/ofscraper/download/shared/utils/log.py @@ -57,21 +57,30 @@ def log_download_progress(media_type): def final_log(username, log=None): + (log or common_globals.log).error( + final_log_text(username) + ) + +def final_log_text(username): if read_args.retriveArgs().metadata: - skipped_word = "media metadata unchanged" - (log or common_globals.log).error( - f"[bold]{username}[/bold] ({format_size(common_globals.total_bytes )}) ({common_globals.photo_count+common_globals.audio_count+common_globals.video_count}" - f" downloads total [{common_globals.video_count} videos, {common_globals.audio_count} audios, {common_globals.photo_count} photos], " - f"{common_globals.forced_skipped} {skipped_word}, {common_globals.skipped} failed)" - ) + return (f"[bold]\\[{username}][/bold] [bold]\\[Action Download][/bold] ({format_size(common_globals.total_bytes )}) ({common_globals.photo_count+common_globals.audio_count+common_globals.video_count}" + f" downloads total [{common_globals.video_count} videos, {common_globals.audio_count} audios, {common_globals.photo_count} photos], " + f"{common_globals.forced_skipped} {skipped_word}, {common_globals.skipped} failed)") else: skipped_word = "skipped" - (log or common_globals.log).error( - f"[bold]{username}[/bold] ({format_size(common_globals.total_bytes )}) ({common_globals.photo_count+common_globals.audio_count+common_globals.video_count}" - f" downloads total [{common_globals.video_count} videos, {common_globals.audio_count} audios, {common_globals.photo_count} photos], " - f"{common_globals.forced_skipped} {skipped_word}, {common_globals.skipped} failed)" - ) + return (f"[bold]\\[{username}][/bold] [bold]\\[Action Download][/bold] ({format_size(common_globals.total_bytes )}) ({common_globals.photo_count+common_globals.audio_count+common_globals.video_count}" + f" downloads total [{common_globals.video_count} videos, {common_globals.audio_count} audios, {common_globals.photo_count} photos], " + f"{common_globals.forced_skipped} {skipped_word}, {common_globals.skipped} failed)") + + +def empty_log(username): + return f"[bold]\\[{username}][/bold] [bold]\\[Action Download][/bold] ({0} photos, {0} videos, {0} audios, {0} skipped, {0} failed)" + + + + + def text_log(username, value=0, fails=0, exists=0, log=None): diff --git a/ofscraper/utils/console.py b/ofscraper/utils/console.py index 132405d3c..446878795 100644 --- a/ofscraper/utils/console.py +++ b/ofscraper/utils/console.py @@ -28,7 +28,7 @@ def get_console(): def get_shared_console(): global shared_console if not shared_console: - shared_console = Console(theme=theme, quiet=quiet) + shared_console = Console(theme=theme, quiet=quiet,markup=True) return shared_console @@ -36,7 +36,7 @@ def get_shared_console(): def get_other_console(): global other_console if not other_console: - other_console = Console(theme=theme) + other_console = Console(theme=theme,markup=True) return other_console diff --git a/ofscraper/utils/logs/classes.py b/ofscraper/utils/logs/classes.py index c5cc4d35a..9f24fd344 100644 --- a/ofscraper/utils/logs/classes.py +++ b/ofscraper/utils/logs/classes.py @@ -8,6 +8,7 @@ import ofscraper.utils.constants as constants import ofscraper.utils.dates as dates_manager import ofscraper.utils.logs.helpers as helpers +from rich.text import Text class PipeHandler(logging.Handler): @@ -163,7 +164,6 @@ def emit(self, record): self._url = record return log_entry = self.format(record) - log_entry = re.sub("\[bold\]|\[/bold\]", "**", log_entry) log_entry = f"{log_entry}\n\n" if constants.getattr("DISCORD_ASYNC"): self._tasks.append(self.loop.create_task(self._async_emit(log_entry))) @@ -261,11 +261,22 @@ def format(self, record): return self._filter(original) +class DiscordFormatter(SensitiveFormatter): + """Formatter that removes sensitive information in logs.""" + + @staticmethod + def _filter(s): + t = SensitiveFormatter._filter(s) + s=re.sub("\\\\+","",t) + s=re.sub(r"\[(bold|/?bold(?:\s\w+\s\w+)?)\]","**",s) + return s + + + class LogFileFormatter(SensitiveFormatter): """Formatter that removes sensitive information in logs.""" @staticmethod def _filter(s): s = SensitiveFormatter._filter(s) - s = re.sub("\[bold\]|\[/bold\]", "", s) - return s + return Text.from_markup(Text(s).plain).plain diff --git a/ofscraper/utils/logs/other.py b/ofscraper/utils/logs/other.py index b7eefe373..9ef173b48 100644 --- a/ofscraper/utils/logs/other.py +++ b/ofscraper/utils/logs/other.py @@ -171,7 +171,7 @@ def init_other_logger(name): # #discord cord = log_class.DiscordHandler() cord.setLevel(log_helpers.getLevel(read_args.retriveArgs().discord)) - cord.setFormatter(log_class.SensitiveFormatter("%(message)s")) + cord.setFormatter(log_class.DiscordFormatter("%(message)s")) # console log.addHandler(cord) if settings.get_log_level() != "OFF": diff --git a/ofscraper/utils/logs/stdout.py b/ofscraper/utils/logs/stdout.py index b0484779e..4eaa3ba94 100644 --- a/ofscraper/utils/logs/stdout.py +++ b/ofscraper/utils/logs/stdout.py @@ -41,15 +41,16 @@ def logger_process(input_, name=None, stop_count=1, event=None): # check for shutdown if event and event.is_set(): break - if message == "None": + if hasattr(message, "message") and message.message!="None": + log.handle(message) + elif hasattr(message, "message") and message.message=="None": count = count + 1 continue - if message.message == "None": + elif message != "None": + log.handle(message) + elif message == "None": count = count + 1 continue - if message.message != "None": - # log the message - log.handle(message) if count == stop_count: break except queue.Empty: