diff --git a/.gitignore b/.gitignore index 86fee46c7..994ff484c 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ share/python-wheels/ *.egg MANIFEST + # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. @@ -147,3 +148,4 @@ core doc* out* *.txt +*.log \ No newline at end of file diff --git a/ofscraper/__main__.py b/ofscraper/__main__.py index 7472f78fb..928db0b95 100755 --- a/ofscraper/__main__.py +++ b/ofscraper/__main__.py @@ -1,13 +1,13 @@ #!/root/OF-Scraper/.venv/bin/python import multiprocessing -import ofscraper.start as start +import ofscraper.runner.load as load import ofscraper.utils.system.system as system def main(): if system.get_parent(): - start.main() + load.main() if __name__ == "__main__": diff --git a/ofscraper/actions/like.py b/ofscraper/actions/like.py index 161a5874d..e1b7f7f89 100644 --- a/ofscraper/actions/like.py +++ b/ofscraper/actions/like.py @@ -42,11 +42,11 @@ sem = semaphoreDelayed(1) log = logging.getLogger("shared") -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args def get_posts(model_id, username): - args = global_args.getArgs() + args = read_args.retriveArgs() pinned_posts = [] timeline_posts = [] archived_posts = [] diff --git a/ofscraper/actions/scraper.py b/ofscraper/actions/scraper.py index d44515285..6d2256f51 100644 --- a/ofscraper/actions/scraper.py +++ b/ofscraper/actions/scraper.py @@ -24,7 +24,8 @@ import ofscraper.classes.posts as posts_ import ofscraper.db.operations as operations import ofscraper.filters.media.main as filters -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.areas as areas +import ofscraper.utils.args.read as read_args import ofscraper.utils.cache as cache import ofscraper.utils.context.stdout as stdout import ofscraper.utils.system.free as free @@ -69,8 +70,8 @@ def process_messages(model_id, username): # Update after database cache.set( "{model_id}_scrape_messages", - global_args.getArgs().after is not None - and global_args.getArgs().after != 0, + read_args.retriveArgs().after is not None + and read_args.retriveArgs().after != 0, ) return list(filter(lambda x: isinstance(x, media.Media), output)) @@ -218,7 +219,9 @@ def process_timeline_posts(model_id, username, individual=False): username=username, downloaded=False, ) - cache.set("{model_id}_scrape_timeline", global_args.getArgs().after is not None) + cache.set( + "{model_id}_scrape_timeline", read_args.retriveArgs().after is not None + ) return list(filter(lambda x: isinstance(x, media.Media), output)) @@ -264,7 +267,9 @@ def process_archived_posts(model_id, username): username=username, downloaded=False, ) - cache.set("{model_id}_scrape_archived", global_args.getArgs().after is not None) + cache.set( + "{model_id}_scrape_archived", read_args.retriveArgs().after is not None + ) return list(filter(lambda x: isinstance(x, media.Media), output)) @@ -406,10 +411,10 @@ def process_labels(model_id, username): labels_ = ( labels_ - if not global_args.getArgs().label + if not read_args.retriveArgs().label else list( filter( - lambda x: x.get("name").lower() in global_args.getArgs().label, + lambda x: x.get("name").lower() in read_args.retriveArgs().label, labels_, ) ) @@ -461,7 +466,7 @@ def process_areas(ele, model_id) -> list: labels_dicts = [] username = ele.name - final_post_areas = set(args_.get_download_area()) + final_post_areas = set(areas.get_download_area()) if "Profile" in final_post_areas: profile_dicts = process_profile(username) if "Pinned" in final_post_areas: diff --git a/ofscraper/api/archive.py b/ofscraper/api/archive.py index 2a33ac1fb..cf1519ab9 100644 --- a/ofscraper/api/archive.py +++ b/ofscraper/api/archive.py @@ -29,7 +29,7 @@ import ofscraper.classes.sessionbuilder as sessionbuilder import ofscraper.db.operations as operations -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args import ofscraper.utils.cache as cache import ofscraper.utils.console as console import ofscraper.utils.constants as constants @@ -52,7 +52,8 @@ async def scrape_archived_posts( attempt.set(0) sem = semaphoreDelayed(constants.getattr("AlT_SEM")) if timestamp and ( - float(timestamp) > (global_args.getArgs().before or arrow.now()).float_timestamp + float(timestamp) + > (read_args.retriveArgs().before or arrow.now()).float_timestamp ): return [] if timestamp: @@ -183,7 +184,7 @@ async def get_archived_media(model_id, username, forced_after=None, rescan=None) page_count = 0 setCache = ( True - if (global_args.getArgs().after == 0 or not global_args.getArgs().after) + if (read_args.retriveArgs().after == 0 or not read_args.retriveArgs().after) else False ) @@ -195,7 +196,7 @@ async def get_archived_media(model_id, username, forced_after=None, rescan=None) operations.get_archived_postinfo( model_id=model_id, username=username ) - if not global_args.getArgs().no_cache + if not read_args.retriveArgs().no_cache else [] ) @@ -212,7 +213,7 @@ async def get_archived_media(model_id, username, forced_after=None, rescan=None) rescan = ( rescan or cache.get("{model_id}_scrape_archived") - and not global_args.getArgs().after + and not read_args.retriveArgs().after ) after = after = ( 0 if rescan else forced_after or get_after(model_id, username) @@ -245,8 +246,8 @@ async def get_archived_media(model_id, username, forced_after=None, rescan=None) required_ids=set( list(map(lambda x: x[0], splitArrays[0])) ), - timestamp=global_args.getArgs().after.float_timestamp - if global_args.getArgs().after + timestamp=read_args.retriveArgs().after.float_timestamp + if read_args.retriveArgs().after else None, ) ) @@ -325,7 +326,7 @@ async def get_archived_media(model_id, username, forced_after=None, rescan=None) ) ) log.debug(f"[bold]Archived Count without Dupes[/bold] {len(unduped)} found") - if setCache and not global_args.getArgs().after: + if setCache and not read_args.retriveArgs().after: newCheck = {} for post in cache.get(f"archived_check_{model_id}", []) + list( unduped.values() @@ -342,8 +343,8 @@ async def get_archived_media(model_id, username, forced_after=None, rescan=None) def get_after(model_id, username): - if global_args.getArgs().after: - return global_args.getArgs().after.float_timestamp + if read_args.retriveArgs().after: + return read_args.retriveArgs().after.float_timestamp curr = operations.get_archived_media(model_id=model_id, username=username) if cache.get(f"{model_id}_scrape_archived"): log.debug( diff --git a/ofscraper/api/lists.py b/ofscraper/api/lists.py index 43a9ae705..2a229b43b 100644 --- a/ofscraper/api/lists.py +++ b/ofscraper/api/lists.py @@ -27,7 +27,7 @@ ) import ofscraper.classes.sessionbuilder as sessionbuilder -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args import ofscraper.utils.console as console import ofscraper.utils.constants as constants from ofscraper.classes.semaphoreDelayed import semaphoreDelayed @@ -42,13 +42,15 @@ async def get_otherlist(): out = [] if ( - len(global_args.getArgs().user_list) >= 2 + len(read_args.retriveArgs().user_list) >= 2 or constants.getattr("OFSCRAPER_RESERVED_LIST") - not in global_args.getArgs().user_list + not in read_args.retriveArgs().user_list ): out.extend(await get_lists()) out = list( - filter(lambda x: x.get("name").lower() in global_args.getArgs().user_list, out) + filter( + lambda x: x.get("name").lower() in read_args.retriveArgs().user_list, out + ) ) log.debug( f"User lists found on profile {list(map(lambda x:x.get('name').lower(),out))}" @@ -59,10 +61,12 @@ async def get_otherlist(): @run async def get_blacklist(): out = [] - if len(global_args.getArgs().black_list) >= 1: + if len(read_args.retriveArgs().black_list) >= 1: out.extend(await get_lists()) out = list( - filter(lambda x: x.get("name").lower() in global_args.getArgs().black_list, out) + filter( + lambda x: x.get("name").lower() in read_args.retriveArgs().black_list, out + ) ) log.debug( f"Black lists found on profile {list(map(lambda x:x.get('name').lower(),out))}" diff --git a/ofscraper/api/messages.py b/ofscraper/api/messages.py index 89ca06ba7..2a662be33 100644 --- a/ofscraper/api/messages.py +++ b/ofscraper/api/messages.py @@ -29,7 +29,7 @@ import ofscraper.classes.sessionbuilder as sessionbuilder import ofscraper.db.operations as operations -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args import ofscraper.utils.cache as cache import ofscraper.utils.console as console import ofscraper.utils.constants as constants @@ -73,7 +73,7 @@ async def get_messages(model_id, username, forced_after=None, rescan=None): async with sessionbuilder.sessionBuilder() as c: oldmessages = ( operations.get_messages_data(model_id=model_id, username=username) - if not global_args.getArgs().no_cache + if not read_args.retriveArgs().no_cache else [] ) log.trace( @@ -97,11 +97,11 @@ async def get_messages(model_id, username, forced_after=None, rescan=None): {"date": arrow.now().float_timestamp, "id": None} ] + oldmessages - before = (global_args.getArgs().before or arrow.now()).float_timestamp + before = (read_args.retriveArgs().before or arrow.now()).float_timestamp rescan = ( rescan or cache.get("{model_id}_scrape_messages") - and not global_args.getArgs().after + and not read_args.retriveArgs().after ) after = after = ( 0 if rescan else forced_after or get_after(model_id, username) @@ -441,8 +441,8 @@ def get_individual_post(model_id, postid, c=None): def get_after(model_id, username): - if global_args.getArgs().after: - return global_args.getArgs().after.float_timestamp + if read_args.retriveArgs().after: + return read_args.retriveArgs().after.float_timestamp if cache.get(f"{model_id}_scrape_messages"): log.debug( "Used after previously scraping entire timeline to make sure content is not missing" diff --git a/ofscraper/api/pinned.py b/ofscraper/api/pinned.py index 966662326..2b2e36e22 100644 --- a/ofscraper/api/pinned.py +++ b/ofscraper/api/pinned.py @@ -28,7 +28,7 @@ ) import ofscraper.classes.sessionbuilder as sessionbuilder -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args import ofscraper.utils.console as console import ofscraper.utils.constants as constants from ofscraper.classes.semaphoreDelayed import semaphoreDelayed @@ -49,7 +49,8 @@ async def scrape_pinned_posts(c, model_id, progress, timestamp=None, count=0) -> attempt.set(0) if timestamp and ( - float(timestamp) > (global_args.getArgs().before or arrow.now()).float_timestamp + float(timestamp) + > (read_args.retriveArgs().before or arrow.now()).float_timestamp ): return [] url = constants.getattr("timelinePinnedEP").format(model_id, count) @@ -172,8 +173,8 @@ async def get_pinned_post(model_id): c, model_id, job_progress, - timestamp=global_args.getArgs().after.float_timestamp - if global_args.getArgs().after + timestamp=read_args.retriveArgs().after.float_timestamp + if read_args.retriveArgs().after else None, ) ) diff --git a/ofscraper/api/subscriptions.py b/ofscraper/api/subscriptions.py index ed211f083..2898103eb 100644 --- a/ofscraper/api/subscriptions.py +++ b/ofscraper/api/subscriptions.py @@ -26,7 +26,7 @@ ) import ofscraper.classes.sessionbuilder as sessionbuilder -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args import ofscraper.utils.constants as constants from ofscraper.classes.semaphoreDelayed import semaphoreDelayed from ofscraper.utils.context.run_async import run @@ -69,16 +69,17 @@ async def activeHelper(subscribe_count, c): global new_tasks if ( - constants.getattr("OFSCRAPER_RESERVED_LIST") in global_args.getArgs().black_list + constants.getattr("OFSCRAPER_RESERVED_LIST") + in read_args.retriveArgs().black_list or constants.getattr("OFSCRAPER_ACTIVE_LIST") - in global_args.getArgs().black_list + in read_args.retriveArgs().black_list ): return [] if ( constants.getattr("OFSCRAPER_RESERVED_LIST") - not in global_args.getArgs().user_list + not in read_args.retriveArgs().user_list and constants.getattr("OFSCRAPER_ACTIVE_LIST") - not in global_args.getArgs().user_list + not in read_args.retriveArgs().user_list ): return [] funct = scrape_subscriptions_active @@ -111,16 +112,17 @@ async def expiredHelper(subscribe_count, c): global new_tasks if ( - constants.getattr("OFSCRAPER_RESERVED_LIST") in global_args.getArgs().black_list + constants.getattr("OFSCRAPER_RESERVED_LIST") + in read_args.retriveArgs().black_list or constants.getattr("OFSCRAPER_EXPIRED_LIST") - in global_args.getArgs().black_list + in read_args.retriveArgs().black_list ): return [] if ( constants.getattr("OFSCRAPER_RESERVED_LIST") - not in global_args.getArgs().user_list + not in read_args.retriveArgs().user_list and constants.getattr("OFSCRAPER_EXPIRED_LIST") - not in global_args.getArgs().user_list + not in read_args.retriveArgs().user_list ): return [] funct = scrape_subscriptions_disabled diff --git a/ofscraper/api/timeline.py b/ofscraper/api/timeline.py index 78e069c85..d876928e3 100644 --- a/ofscraper/api/timeline.py +++ b/ofscraper/api/timeline.py @@ -29,7 +29,7 @@ import ofscraper.classes.sessionbuilder as sessionbuilder import ofscraper.db.operations as operations -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args import ofscraper.utils.cache as cache import ofscraper.utils.console as console import ofscraper.utils.constants as constants @@ -50,7 +50,8 @@ async def scrape_timeline_posts( attempt.set(0) if timestamp and ( - float(timestamp) > (global_args.getArgs().before or arrow.now()).float_timestamp + float(timestamp) + > (read_args.retriveArgs().before or arrow.now()).float_timestamp ): return [] if timestamp: @@ -181,7 +182,7 @@ async def get_timeline_media(model_id, username, forced_after=None, rescan=None) min_posts = 50 responseArray = [] page_count = 0 - if not global_args.getArgs().no_cache: + if not read_args.retriveArgs().no_cache: oldtimeline = operations.get_timeline_postdates( model_id=model_id, username=username ) @@ -198,7 +199,8 @@ async def get_timeline_media(model_id, username, forced_after=None, rescan=None) oldtimeline = list(filter(lambda x: x != None, oldtimeline)) postedAtArray = sorted(oldtimeline) rescan = ( - cache.get("{model_id}_scrape_timeline") and not global_args.getArgs().after + cache.get("{model_id}_scrape_timeline") + and not read_args.retriveArgs().after ) after = after = 0 if rescan else forced_after or get_after(model_id, username) @@ -333,8 +335,8 @@ def get_individual_post(id, c=None): def get_after(model_id, username): - if global_args.getArgs().after: - return global_args.getArgs().after.float_timestamp + if read_args.retriveArgs().after: + return read_args.retriveArgs().after.float_timestamp curr = operations.get_timeline_media(model_id=model_id, username=username) if cache.get(f"{model_id}_scrape_timeline"): log.debug( diff --git a/ofscraper/classes/models.py b/ofscraper/classes/models.py index 48c90b491..86ca0e21d 100644 --- a/ofscraper/classes/models.py +++ b/ofscraper/classes/models.py @@ -1,6 +1,6 @@ import arrow -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args FORMAT = "YYYY-MM-DD" @@ -129,7 +129,7 @@ def active(self): return True elif self.renewed: return True - elif arrow.get(self.final_expired) > global_args.getArgs().date_now: + elif arrow.get(self.final_expired) > read_args.retriveArgs().date_now: return True return False diff --git a/ofscraper/classes/placeholder.py b/ofscraper/classes/placeholder.py index 8a2505adc..a93e9e87d 100644 --- a/ofscraper/classes/placeholder.py +++ b/ofscraper/classes/placeholder.py @@ -7,7 +7,7 @@ import ofscraper.api.me as me import ofscraper.filters.models.selector as selector -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args import ofscraper.utils.cache as cache import ofscraper.utils.config.custom as custom_ import ofscraper.utils.config.data as data @@ -234,6 +234,7 @@ def createfilename(self, ele, username, model_id, ext): filename = ele.final_filename self._variables.update({"filename": filename}) self._variables.update({"file_name": filename}) + self._variables.update({"ext": ext}) text = ele.file_text self._variables.update({"text": text}) self.add_common_variables(ele, username, model_id) @@ -278,11 +279,11 @@ def _addcount(self, ele, out): return out def set_final_path(self): - if (global_args.getArgs().original or data.get_truncation()) is False: + if (read_args.retriveArgs().original or data.get_truncation()) is False: self._final_path = pathlib.Path(self.mediadir, f"{self.filename}") - elif global_args.getArgs().original is False: + elif read_args.retriveArgs().original is False: self._final_path = pathlib.Path(self.mediadir, f"{self.filename}") - elif global_args.getArgs().original is True or data.get_truncation() is True: + elif read_args.retriveArgs().original is True or data.get_truncation() is True: self._final_path = paths.truncate( pathlib.Path(self.mediadir, f"{self.filename}") ) diff --git a/ofscraper/commands/check.py b/ofscraper/commands/check.py index d946e15c3..ebfd57328 100644 --- a/ofscraper/commands/check.py +++ b/ofscraper/commands/check.py @@ -20,7 +20,8 @@ import ofscraper.commands.manual as manual import ofscraper.db.operations as operations import ofscraper.download.downloadnormal as downloadnormal -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args +import ofscraper.utils.args.write as write_args import ofscraper.utils.auth as auth import ofscraper.utils.cache as cache import ofscraper.utils.console as console_ @@ -149,7 +150,7 @@ def post_checker(): oldtimeline = cache.get(f"timeline_check_{model_id}", default=[]) user_dict[user_name] = {} user_dict[user_name] = user_dict[user_name] or [] - if len(oldtimeline) > 0 and not global_args.getArgs().force: + if len(oldtimeline) > 0 and not read_args.retriveArgs().force: user_dict[user_name].extend(oldtimeline) else: user_dict[user_name] = {} @@ -164,7 +165,7 @@ def post_checker(): cache.close() oldarchive = cache.get(f"archived_check_{model_id}", default=[]) - if len(oldarchive) > 0 and not global_args.getArgs().force: + if len(oldarchive) > 0 and not read_args.retriveArgs().force: user_dict[user_name].extend(oldarchive) else: data = archive.get_archived_media(model_id, user_name, forced_after=0) @@ -212,15 +213,15 @@ def post_checker(): def reset_url(): # clean up args once check modes are ready to launch - args = global_args.getArgs() + args = read_args.retriveArgs() argdict = vars(args) if argdict.get("url"): - global_args.getArgs().url = None + read_args.retriveArgs().url = None if argdict.get("file"): - global_args.getArgs().file = None + read_args.retriveArgs().file = None if argdict.get("username"): - global_args.getArgs().username = None - args_.changeargs(args) + read_args.retriveArgs().username = None + write_args.setArgs(args) def set_count(ROWS): @@ -250,7 +251,7 @@ def message_checker(): oldmessages = cache.get(f"message_check_{model_id}", default=[]) log.debug(f"Number of messages in cache {len(oldmessages)}") - if len(oldmessages) > 0 and not global_args.getArgs().force: + if len(oldmessages) > 0 and not read_args.retriveArgs().force: messages = oldmessages else: messages = messages_.get_messages(model_id, user_name, forced_after=0) @@ -262,7 +263,7 @@ def message_checker(): oldpaid = cache.get(f"purchased_check_{model_id}", default=[]) paid = None # paid content - if len(oldpaid) > 0 and not global_args.getArgs().force: + if len(oldpaid) > 0 and not read_args.retriveArgs().force: paid = oldpaid else: paid = paid_.get_paid_posts(user_name, model_id) @@ -292,14 +293,14 @@ def purchase_checker(): user_dict = {} auth.make_headers(auth.read_auth()) ROWS = [] - for user_name in global_args.getArgs().username: + for user_name in read_args.retriveArgs().username: user_name = profile.scrape_profile(user_name)["username"] user_dict[user_name] = user_dict.get(user_name, []) model_id = profile.get_id(user_name) oldpaid = cache.get(f"purchased_check_{model_id}", default=[]) paid = None - if len(oldpaid) > 0 and not global_args.getArgs().force: + if len(oldpaid) > 0 and not read_args.retriveArgs().force: paid = oldpaid else: paid = paid_.get_paid_posts(user_name, model_id) @@ -320,7 +321,7 @@ def purchase_checker(): def stories_checker(): user_dict = {} ROWS = [] - for user_name in global_args.getArgs().username: + for user_name in read_args.retriveArgs().username: user_name = profile.scrape_profile(user_name)["username"] user_dict[user_name] = user_dict.get(user_name, []) model_id = profile.get_id(user_name) @@ -347,8 +348,8 @@ def stories_checker(): def url_helper(): out = [] - out.extend(global_args.getArgs().file or []) - out.extend(global_args.getArgs().url or []) + out.extend(read_args.retriveArgs().file or []) + out.extend(read_args.retriveArgs().url or []) return map(lambda x: x.strip(), out) @@ -381,7 +382,7 @@ def get_paid_ids(model_id, user_name): oldpaid = cache.get(f"purchased_check_{model_id}", default=[]) paid = None - if len(oldpaid) > 0 and not global_args.getArgs().force: + if len(oldpaid) > 0 and not read_args.retriveArgs().force: paid = oldpaid else: paid = paid_.get_paid_posts(user_name, model_id) diff --git a/ofscraper/commands/manual.py b/ofscraper/commands/manual.py index 3a7df24de..0851334fc 100644 --- a/ofscraper/commands/manual.py +++ b/ofscraper/commands/manual.py @@ -11,7 +11,8 @@ import ofscraper.classes.sessionbuilder as sessionbuilder import ofscraper.db.operations as operations import ofscraper.download.download as download -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args +import ofscraper.utils.args.write as write_args import ofscraper.utils.constants as constants import ofscraper.utils.system.network as network @@ -21,9 +22,9 @@ def manual_download(urls=None): network.check_cdm() media_dict = get_media_from_urls(urls) log.debug(f"Media dict length {len(list(media_dict.values()))}") - args = global_args.getArgs() + args = read_args.retriveArgs() args.dupe = True - args_.changeargs(args) + write_args.setArgs(args) for value in media_dict.values(): if len(value) == 0: @@ -40,9 +41,9 @@ def manual_download(urls=None): def get_media_from_urls(urls): - args = global_args.getArgs() + args = read_args.retriveArgs() args.dupe = True - args_.changeargs(args) + write_args.setArgs(args) user_name_dict = {} id_dict = {} with sessionbuilder.sessionBuilder(backend="httpx") as c: @@ -169,7 +170,7 @@ def get_info(url): def url_helper(urls): - args = global_args.getArgs() + args = read_args.retriveArgs() args = vars(args) out = [] out.extend(args.get("file", []) or []) diff --git a/ofscraper/commands/picker.py b/ofscraper/commands/picker.py index 424eaac05..f3935034b 100644 --- a/ofscraper/commands/picker.py +++ b/ofscraper/commands/picker.py @@ -1,11 +1,11 @@ import ofscraper.commands.check as check import ofscraper.commands.manual as manual import ofscraper.commands.scraper as scraper -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args def pick(): - args = global_args.getArgs() + args = read_args.retriveArgs() if args.command == "post_check": check.post_checker() elif args.command == "msg_check": diff --git a/ofscraper/commands/scraper.py b/ofscraper/commands/scraper.py index 1d27415d3..9b4e53896 100755 --- a/ofscraper/commands/scraper.py +++ b/ofscraper/commands/scraper.py @@ -33,7 +33,7 @@ import ofscraper.filters.models.selector as userselector import ofscraper.prompts.prompts as prompts import ofscraper.utils.actions as actions -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args import ofscraper.utils.auth as auth import ofscraper.utils.config.config as config_ import ofscraper.utils.config.data as data @@ -41,6 +41,7 @@ import ofscraper.utils.constants as constants import ofscraper.utils.context.exit as exit import ofscraper.utils.context.stdout as stdout +import ofscraper.utils.dates as dates import ofscraper.utils.logger as logger import ofscraper.utils.paths.check as check import ofscraper.utils.paths.paths as paths @@ -56,16 +57,16 @@ def add_selected_areas(count=None): functs = [] count = count or 0 - if "download" in global_args.getArgs().action: + if "download" in read_args.retriveArgs().action: actions.set_download_area() functs.append(process_post) - if "like" in global_args.getArgs().action: + if "like" in read_args.retriveArgs().action: actions.set_like_area() functs.append(process_like) - if "unlike" in global_args.getArgs().action: + if "unlike" in read_args.retriveArgs().action: actions.set_like_area() functs.append(process_unlike) - if global_args.getArgs().scrape_paid: + if read_args.retriveArgs().scrape_paid: functs.append(scrape_paid) return functs @@ -148,7 +149,7 @@ def main_prompt_action(result_main_prompt): @exit.exit_wrapper def process_post(): - if global_args.getArgs().users_first: + if read_args.retriveArgs().users_first: process_post_user_first() else: normal_post_process() @@ -175,9 +176,9 @@ def process_post_user_first(): log.info(f"Progress {count+1}/{length} model") if constants.getattr("SHOW_AVATAR") and ele.avatar: log.warning(f"Avatar : {ele.avatar}") - if global_args.getArgs().posts: + if read_args.retriveArgs().posts: log.info( - f"Getting {','.join(global_args.getArgs().posts)} for [bold]{ele.name}[/bold]\n[bold]Subscription Active:[/bold] {ele.active}" + f"Getting {','.join(read_args.retriveArgs().posts)} for [bold]{ele.name}[/bold]\n[bold]Subscription Active:[/bold] {ele.active}" ) try: model_id = ele.id @@ -234,9 +235,9 @@ def normal_post_process(): log.warning(f"Progress {count+1}/{length} model") if constants.getattr("SHOW_AVATAR") and ele.avatar: log.warning(f"Avatar : {ele.avatar}") - if global_args.getArgs().posts: + if read_args.retriveArgs().posts: log.warning( - f"Getting {','.join(global_args.getArgs().posts)} for [bold]{ele.name}[/bold]\n[bold]Subscription Active:[/bold] {ele.active}" + f"Getting {','.join(read_args.retriveArgs().posts)} for [bold]{ele.name}[/bold]\n[bold]Subscription Active:[/bold] {ele.active}" ) try: model_id = ele.id @@ -251,7 +252,7 @@ def normal_post_process(): log.traceback_(f"failed with exception: {e}") log.traceback_(traceback.format_exc()) - if global_args.getArgs().scrape_paid: + if read_args.retriveArgs().scrape_paid: user_dict = {} [ user_dict.update( @@ -346,7 +347,7 @@ def process_unlike(): # Adds a function to the job queue def set_schedule(*functs): - schedule.every(global_args.getArgs().daemon).minutes.do(schedule_helper, functs) + schedule.every(read_args.retriveArgs().daemon).minutes.do(schedule_helper, functs) while len(schedule.jobs) > 0: schedule.run_pending() time.sleep(30) @@ -367,7 +368,7 @@ def daemon_run_helper(*functs): jobqueue = queue.Queue() worker_thread = None [jobqueue.put(funct) for funct in functs] - if global_args.getArgs().output == "PROMPT": + if read_args.retriveArgs().output == "PROMPT": log.info(f"[bold]silent-mode on[/bold]") check_auth() log.info(f"[bold]daemon mode on[/bold]") @@ -409,14 +410,14 @@ def run_helper(*functs): global jobqueue jobqueue = queue.Queue() [jobqueue.put(funct) for funct in functs] - if global_args.getArgs().output == "PROMPT": + if read_args.retriveArgs().output == "PROMPT": log.info(f"[bold]silent-mode on[/bold]") try: for _ in functs: job_func = jobqueue.get() job_func() jobqueue.task_done() - args_.clearDate() + dates.resetLogdate() except KeyboardInterrupt: try: with exit.DelayedKeyboardInterrupt(): @@ -487,7 +488,7 @@ def scrape_context_manager(): def print_start(): with stdout.lowstdout(): console.get_shared_console().print( - f"[bold green] Welcome to OF-Scraper Version {global_args.getArgs().version}[/bold green]" + f"[bold green] Welcome to OF-Scraper Version {read_args.retriveArgs().version}[/bold green]" ) @@ -527,7 +528,7 @@ def scrapper(): os.system("color") global selectedusers selectedusers = None - args = global_args.getArgs() + args = read_args.retriveArgs() if args.daemon: if len(args.action) == 0 and not args.scrape_paid: prompts.action_prompt() diff --git a/ofscraper/const/config/paths.py b/ofscraper/const/config/paths.py index 8908d4b21..cea86ed0d 100644 --- a/ofscraper/const/config/paths.py +++ b/ofscraper/const/config/paths.py @@ -1,7 +1,7 @@ import pathlib import ofscraper.const.constants as consts -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args def get_config_home(): @@ -9,7 +9,7 @@ def get_config_home(): def get_config_path(): - configPath = global_args.getArgs().config + configPath = read_args.retriveArgs().config defaultPath = pathlib.Path.home() / consts.configPath / consts.configFile ofscraperHome = pathlib.Path.home() / consts.configPath diff --git a/ofscraper/db/operations.py b/ofscraper/db/operations.py index 26c106717..2fef5a5ac 100644 --- a/ofscraper/db/operations.py +++ b/ofscraper/db/operations.py @@ -24,9 +24,9 @@ import ofscraper.classes.placeholder as placeholder import ofscraper.utils.cache as cache +import ofscraper.utils.constants as constants import ofscraper.utils.context.exit as exit import ofscraper.utils.paths.common as common_paths -from ofscraper.const.constants import DBINTERVAL from ofscraper.utils.context.run_async import run from ..db import queries @@ -565,7 +565,7 @@ def create_backup(model_id, username): now = arrow.now().float_timestamp last = cache.get(f"{username}_{model_id}_db_backup", default=now) - if now - last > DBINTERVAL and database_path.exists(): + if now - last > constants.getattr("DBINTERVAL") and database_path.exists(): database_copy = placeholder.Placeholders().databasePathCopyHelper( model_id, username ) diff --git a/ofscraper/download/alt_download.py b/ofscraper/download/alt_download.py index 97889ef5b..63c24dc0e 100644 --- a/ofscraper/download/alt_download.py +++ b/ofscraper/download/alt_download.py @@ -32,7 +32,7 @@ import ofscraper.db.operations as operations import ofscraper.download.common as common import ofscraper.download.keyhelpers as keyhelpers -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args import ofscraper.utils.cache as cache import ofscraper.utils.config.data as data import ofscraper.utils.constants as constants @@ -60,7 +60,7 @@ async def alt_download(c, ele, username, model_id, progress): common.log.debug(f"{get_medialog(ele)} Downloading with protected media downloader") common.log.debug(f"{get_medialog(ele)} download url: {get_url_log(ele)}") - if global_args.getArgs().metadata: + if read_args.retriveArgs().metadata: return await metadata( c, ele, @@ -306,7 +306,7 @@ async def alt_download_datahandler(item, total, l, ele, progress, placeholderObj pathstr = str(placeholderObj.tempfilename) downloadprogress = ( - data.get_show_downloadprogress() or global_args.getArgs().downloadbars + data.get_show_downloadprogress() or read_args.retriveArgs().downloadbars ) task1 = progress.add_task( @@ -387,7 +387,7 @@ async def alt_download_downloader(item, c, ele, username, model_id, progress): partial(cache.get, f"{item['name']}_headers"), ) pathlib.Path(placeholderObj.tempfilename).unlink(missing_ok=True) if ( - global_args.getArgs().no_auto_resume + read_args.retriveArgs().no_auto_resume or not data.get_part_file_clean() or False ) else None diff --git a/ofscraper/download/alt_downloadbatch.py b/ofscraper/download/alt_downloadbatch.py index 7875a0929..74a66aaeb 100644 --- a/ofscraper/download/alt_downloadbatch.py +++ b/ofscraper/download/alt_downloadbatch.py @@ -23,7 +23,7 @@ import ofscraper.db.operations as operations import ofscraper.download.common as common import ofscraper.download.keyhelpers as keyhelpers -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args import ofscraper.utils.cache as cache import ofscraper.utils.config.data as data import ofscraper.utils.constants as constants @@ -57,7 +57,7 @@ async def alt_download(c, ele, username, model_id): common.innerlog.get().debug( f"{get_medialog(ele)} download url: {get_url_log(ele)}" ) - if global_args.getArgs().metadata: + if read_args.retriveArgs().metadata: sharedPlaceholderObj = placeholder.Placeholders() sharedPlaceholderObj.getmediadir(ele, username, model_id, create=False) sharedPlaceholderObj.createfilename(ele, username, model_id, "mp4") diff --git a/ofscraper/download/common.py b/ofscraper/download/common.py index 90c7ed5c6..a46c403ec 100644 --- a/ofscraper/download/common.py +++ b/ofscraper/download/common.py @@ -43,7 +43,8 @@ import ofscraper.classes.placeholder as placeholder import ofscraper.db.operations as operations -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args +import ofscraper.utils.args.write as write_args import ofscraper.utils.cache as cache import ofscraper.utils.config.data as config_data import ofscraper.utils.console as console_ @@ -122,8 +123,8 @@ def reset_globals(): ) -def setLogDate(args): - args_.changeargs(args) +def setLogDate(date): + dates.setLogDate(date) def get_medialog(ele): @@ -137,9 +138,10 @@ def process_split_globals(pipeCopy, logCopy): log = logCopy -def subProcessVariableInit(argsCopy, pipeCopy, logCopy): +def subProcessVariableInit(date, pipeCopy, logCopy, argsCopy): reset_globals() - setLogDate(argsCopy) + write_args.setArgs(argsCopy) + setLogDate(date) process_split_globals(pipeCopy, logCopy) @@ -173,7 +175,7 @@ async def inner(*args, input_sem=input_sem, **kwargs): def setupProgressBar(multi=False): downloadprogress = ( - config_data.get_show_downloadprogress() or global_args.getArgs().downloadbars + config_data.get_show_downloadprogress() or read_args.retriveArgs().downloadbars ) if not multi: job_progress = Progress( @@ -266,8 +268,10 @@ async def check_forced_skip(ele, *args): total = sum(map(lambda x: int(x), args)) if total == 0: return 0 - file_size_limit = global_args.getArgs().size_max or config_data.get_filesize_limit() - file_size_min = global_args.getArgs().size_min or config_data.get_filesize_limit() + file_size_limit = ( + read_args.retriveArgs().size_max or config_data.get_filesize_limit() + ) + file_size_min = read_args.retriveArgs().size_min or config_data.get_filesize_limit() if int(file_size_limit) > 0 and (int(total) > int(file_size_limit)): ele.mediatype = "forced_skipped" log.debug(f"{get_medialog(ele)} {format_size(total)} over size limit") diff --git a/ofscraper/download/download.py b/ofscraper/download/download.py index ac91f0a90..f2bb78256 100644 --- a/ofscraper/download/download.py +++ b/ofscraper/download/download.py @@ -3,7 +3,7 @@ import ofscraper.db.operations as operations import ofscraper.download.downloadbatch as batchdownloader import ofscraper.download.downloadnormal as normaldownloader -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args import ofscraper.utils.config.data as data import ofscraper.utils.constants as constants import ofscraper.utils.separate as seperate @@ -12,7 +12,7 @@ def medialist_filter(medialist, model_id, username): log = logging.getLogger("shared") - if not global_args.getArgs().dupe: + if not read_args.retriveArgs().dupe: media_ids = set( operations.get_media_ids_downloaded(model_id=model_id, username=username) ) @@ -44,7 +44,7 @@ def download_picker(username, model_id, medialist): len(medialist) >= data.get_download_semaphores() * constants.getattr("DOWNLOAD_THREAD_MIN") ) - and global_args.getArgs().downloadthreads != 0 + and read_args.retriveArgs().downloadthreads != 0 and data.get_threads() != 0 ): return batchdownloader.process_dicts(username, model_id, medialist) diff --git a/ofscraper/download/downloadbatch.py b/ofscraper/download/downloadbatch.py index ea3644c6f..8965a3366 100644 --- a/ofscraper/download/downloadbatch.py +++ b/ofscraper/download/downloadbatch.py @@ -17,12 +17,13 @@ import ofscraper.classes.sessionbuilder as sessionbuilder import ofscraper.download.common as common -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args import ofscraper.utils.config.data as config_data import ofscraper.utils.console as console import ofscraper.utils.constants as constants import ofscraper.utils.context.exit as exit import ofscraper.utils.context.stdout as stdout +import ofscraper.utils.dates as dates import ofscraper.utils.logger as logger import ofscraper.utils.manager as manager_ import ofscraper.utils.system.system as system @@ -82,7 +83,8 @@ def process_dicts(username, model_id, filtered_medialist): logqueues_[i // split_val], otherqueues_[i // split_val], connect_tuples[i][1], - global_args.getArgs(), + dates.getLogDate(), + read_args.retriveArgs(), ), ) for i in range(num_proc) @@ -220,7 +222,7 @@ def process_dicts(username, model_id, filtered_medialist): def queue_process(pipe_, overall_progress, job_progress, task1, total): count = 0 downloadprogress = ( - global_args.getArgs().downloadbars or config_data.get_show_downloadprogress() + read_args.retriveArgs().downloadbars or config_data.get_show_downloadprogress() ) # shared globals @@ -295,7 +297,7 @@ def queue_process(pipe_, overall_progress, job_progress, task1, total): def get_mediasplits(medialist): - user_count = global_args.getArgs().downloadthreads or config_data.get_threads() + user_count = read_args.retriveArgs().downloadthreads or config_data.get_threads() final_count = min(user_count, system.getcpu_count(), len(medialist) // 5) if final_count == 0: final_count = 1 @@ -303,14 +305,15 @@ def get_mediasplits(medialist): def process_dict_starter( - username, model_id, ele, p_logqueue_, p_otherqueue_, pipe_, argsCopy + username, model_id, ele, p_logqueue_, p_otherqueue_, pipe_, date, argsCopy ): subProcessVariableInit( - argsCopy, + date, pipe_, logger.get_shared_logger( main_=p_logqueue_, other_=p_otherqueue_, name=f"shared_{os.getpid()}" ), + argsCopy, ) setpriority() plat = platform.system() diff --git a/ofscraper/download/keyhelpers.py b/ofscraper/download/keyhelpers.py index 219534775..b1adb80bc 100644 --- a/ofscraper/download/keyhelpers.py +++ b/ofscraper/download/keyhelpers.py @@ -20,7 +20,7 @@ import ofscraper.classes.sessionbuilder as sessionbuilder import ofscraper.download.common as common -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args import ofscraper.utils.auth as auth import ofscraper.utils.cache as cache import ofscraper.utils.config.data as data @@ -38,7 +38,7 @@ def setLog(input_): async def un_encrypt(item, c, ele, input_=None): setLog(input_ or common.log) key = None - keymode = global_args.getArgs().key_mode or data.get_key_mode() or "cdrm" + keymode = read_args.retriveArgs().key_mode or data.get_key_mode() or "cdrm" past_key = await asyncio.get_event_loop().run_in_executor( common.cache_thread, partial(cache.get, ele.license) ) diff --git a/ofscraper/download/main_download.py b/ofscraper/download/main_download.py index 71ff3c8ac..45a1715a6 100644 --- a/ofscraper/download/main_download.py +++ b/ofscraper/download/main_download.py @@ -29,7 +29,7 @@ import ofscraper.classes.placeholder as placeholder import ofscraper.db.operations as operations import ofscraper.download.common as common -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args import ofscraper.utils.cache as cache import ofscraper.utils.config.data as data import ofscraper.utils.constants as constants @@ -56,7 +56,7 @@ async def main_download(c, ele, username, model_id, progress): common.log.debug(f"{get_medialog(ele)} Downloading with normal downloader") common.log.debug(f"{get_medialog(ele)} download url: {get_url_log(ele)}") # total may be none if no .part file - if global_args.getArgs().metadata: + if read_args.retriveArgs().metadata: return await metadata( c, ele, @@ -302,7 +302,7 @@ async def inner(): async def main_download_datahandler(r, progress, ele, placeholderObj, total): pathstr = str(placeholderObj.trunicated_filename) downloadprogress = ( - data.get_show_downloadprogress() or global_args.getArgs().downloadbars + data.get_show_downloadprogress() or read_args.retriveArgs().downloadbars ) task1 = progress.add_task( f"{(pathstr[:constants.getattr('PATH_STR_MAX)')] + '....') if len(pathstr) > constants.getattr('PATH_STR_MAX') else pathstr}\n", diff --git a/ofscraper/download/main_downloadbatch.py b/ofscraper/download/main_downloadbatch.py index 91dec251d..b967c139f 100644 --- a/ofscraper/download/main_downloadbatch.py +++ b/ofscraper/download/main_downloadbatch.py @@ -29,7 +29,7 @@ import ofscraper.classes.placeholder as placeholder import ofscraper.db.operations as operations import ofscraper.download.common as common -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args import ofscraper.utils.cache as cache import ofscraper.utils.config.data as data import ofscraper.utils.constants as constants @@ -58,7 +58,7 @@ async def main_download(c, ele, username, model_id): f"{get_medialog(ele)} Downloading with normal batch downloader" ) common.innerlog.get().debug(f"{get_medialog(ele)} download url: {get_url_log(ele)}") - if global_args.getArgs().metadata: + if read_args.retriveArgs().metadata: return await metadata( c, ele, @@ -293,7 +293,7 @@ async def inner(): async def main_download_datahandler(r, ele, total, placeholderObj): pathstr = str(placeholderObj.trunicated_filename) downloadprogress = ( - data.get_show_downloadprogress() or global_args.getArgs().downloadbars + data.get_show_downloadprogress() or read_args.retriveArgs().downloadbars ) try: count = 0 diff --git a/ofscraper/filters/media/helpers.py b/ofscraper/filters/media/helpers.py index 8e0515964..7fa85f136 100644 --- a/ofscraper/filters/media/helpers.py +++ b/ofscraper/filters/media/helpers.py @@ -3,7 +3,7 @@ import arrow -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args import ofscraper.utils.config.data as data log = logging.getLogger("shared") @@ -33,17 +33,18 @@ def timeline_array_filter(posts): undated = list(filter(lambda x: x.get("postedAt") is None, posts)) dated = list(filter(lambda x: x.get("postedAt") is not None, posts)) dated = sorted(dated, key=lambda x: arrow.get(x.get("postedAt"))) - if global_args.getArgs().before: + if read_args.retriveArgs().before: dated = list( filter( - lambda x: arrow.get(x.get("postedAt")) <= global_args.getArgs().before, + lambda x: arrow.get(x.get("postedAt")) + <= read_args.retriveArgs().before, dated, ) ) - if global_args.getArgs().after: + if read_args.retriveArgs().after: dated = list( filter( - lambda x: arrow.get(x.get("postedAt")) >= global_args.getArgs().after, + lambda x: arrow.get(x.get("postedAt")) >= read_args.retriveArgs().after, dated, ) ) @@ -53,7 +54,7 @@ def timeline_array_filter(posts): def posts_type_filter(media): - filtersettings = global_args.getArgs().mediatype or data.get_filter() + filtersettings = read_args.retriveArgs().mediatype or data.get_filter() if isinstance(filtersettings, str): filtersettings = filtersettings.split(",") if isinstance(filtersettings, list): @@ -70,19 +71,19 @@ def posts_type_filter(media): def posts_date_filter(media): - if global_args.getArgs().before: + if read_args.retriveArgs().before: media = list( filter( lambda x: x.postdate is None - or arrow.get(x.postdate) <= global_args.getArgs().before, + or arrow.get(x.postdate) <= read_args.retriveArgs().before, media, ) ) - if global_args.getArgs().after: + if read_args.retriveArgs().after: media = list( filter( lambda x: x.postdate is None - or arrow.get(x.postdate) >= global_args.getArgs().after, + or arrow.get(x.postdate) >= read_args.retriveArgs().after, media, ) ) @@ -90,15 +91,15 @@ def posts_date_filter(media): def post_timed_filter(media): - if global_args.getArgs().timed_only is False: + if read_args.retriveArgs().timed_only is False: return list(filter(lambda x: not x.expires, media)) - elif global_args.getArgs().timed_only is True: + elif read_args.retriveArgs().timed_only is True: return list(filter(lambda x: x.expires, media)) return media def post_user_filter(media): - userfilter = global_args.getArgs().filter + userfilter = read_args.retriveArgs().filter if not userfilter: return media elif not userfilter.islower(): @@ -116,7 +117,7 @@ def post_user_filter(media): def anti_post_user_filter(media): - userfilter = global_args.getArgs().neg_filter + userfilter = read_args.retriveArgs().neg_filter if not userfilter: return media elif not userfilter.islower(): @@ -133,18 +134,18 @@ def anti_post_user_filter(media): def download_type_filter(media): - if global_args.getArgs().protected_only: + if read_args.retriveArgs().protected_only: return list(filter(lambda x: x.mpd is not None, media)) - elif global_args.getArgs().normal_only: + elif read_args.retriveArgs().normal_only: return list(filter(lambda x: x.url is not None, media)) else: return media def mass_msg_filter(media): - if global_args.getArgs().mass_msg is None: + if read_args.retriveArgs().mass_msg is None: return media - elif global_args.getArgs().mass_msg is True: + elif read_args.retriveArgs().mass_msg is True: return list((filter(lambda x: x.mass is True, media))) - elif global_args.getArgs().mass_msg is False: + elif read_args.retriveArgs().mass_msg is False: return list((filter(lambda x: x.mass is False, media))) diff --git a/ofscraper/filters/models/date.py b/ofscraper/filters/models/date.py index 930d603f6..77e343774 100644 --- a/ofscraper/filters/models/date.py +++ b/ofscraper/filters/models/date.py @@ -6,64 +6,66 @@ import arrow -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args def dateFilters(filterusername): log = logging.getLogger("shared") - log.debug(f"Last Seen After Filter: {global_args.getArgs().last_seen_after}") - if global_args.getArgs().last_seen_after: + log.debug(f"Last Seen After Filter: {read_args.retriveArgs().last_seen_after}") + if read_args.retriveArgs().last_seen_after: filterusername = list( filter( - lambda x: x.final_last_seen >= global_args.getArgs().last_seen_after, + lambda x: x.final_last_seen >= read_args.retriveArgs().last_seen_after, filterusername, ) ) log.debug(f"last seen after username count: {len(filterusername)}") - log.debug(f"Last Seen Before Filter: {global_args.getArgs().last_seen_before}") - if global_args.getArgs().last_seen_before: + log.debug(f"Last Seen Before Filter: {read_args.retriveArgs().last_seen_before}") + if read_args.retriveArgs().last_seen_before: filterusername = list( filter( - lambda x: x.final_last_seen <= global_args.getArgs().last_seen_before, + lambda x: x.final_last_seen <= read_args.retriveArgs().last_seen_before, filterusername, ) ) log.debug(f"last seen before username count: {len(filterusername)}") - log.debug(f"Subscribed After Filter: {global_args.getArgs().subscribed_after}") - if global_args.getArgs().subscribed_after: + log.debug(f"Subscribed After Filter: {read_args.retriveArgs().subscribed_after}") + if read_args.retriveArgs().subscribed_after: filterusername = list( filter( lambda x: arrow.get(x.subscribed) - >= global_args.getArgs().subscribed_after, + >= read_args.retriveArgs().subscribed_after, filterusername, ) ) log.debug(f"subscribed after username count: {len(filterusername)}") - log.debug(f"Subscribed Before Filter: {global_args.getArgs().subscribed_before}") - if global_args.getArgs().subscribed_before: + log.debug(f"Subscribed Before Filter: {read_args.retriveArgs().subscribed_before}") + if read_args.retriveArgs().subscribed_before: filterusername = list( filter( - lambda x: x.final_last_seen <= global_args.getArgs().subscribed_before, + lambda x: x.final_last_seen + <= read_args.retriveArgs().subscribed_before, filterusername, ) ) log.debug(f"subscribed before username count: {len(filterusername)}") - log.debug(f"Expired After Filter: {global_args.getArgs().expired_after}") - if global_args.getArgs().expired_after: + log.debug(f"Expired After Filter: {read_args.retriveArgs().expired_after}") + if read_args.retriveArgs().expired_after: filterusername = list( filter( - lambda x: arrow.get(x.expired) >= global_args.getArgs().expired_after, + lambda x: arrow.get(x.expired) >= read_args.retriveArgs().expired_after, filterusername, ) ) log.debug(f"expired after username count: {len(filterusername)}") - log.debug(f"Expired Before Filter: {global_args.getArgs().expired_before}") - if global_args.getArgs().expired_before: + log.debug(f"Expired Before Filter: {read_args.retriveArgs().expired_before}") + if read_args.retriveArgs().expired_before: filterusername = list( filter( - lambda x: arrow.get(x.expired) <= global_args.getArgs().expired_before, + lambda x: arrow.get(x.expired) + <= read_args.retriveArgs().expired_before, filterusername, ) ) diff --git a/ofscraper/filters/models/flags.py b/ofscraper/filters/models/flags.py index 127d1359c..d09f07a06 100644 --- a/ofscraper/filters/models/flags.py +++ b/ofscraper/filters/models/flags.py @@ -1,46 +1,46 @@ import logging -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args def promoFilterHelper(filterusername): log = logging.getLogger("shared") - log.debug(f"Promo Flag: {global_args.getArgs().promo}") - if global_args.getArgs().promo: + log.debug(f"Promo Flag: {read_args.retriveArgs().promo}") + if read_args.retriveArgs().promo: filterusername = list( filter(lambda x: x.lowest_promo_claim is not None, filterusername) ) log.debug(f"Promo Flag Yes Count: {len(filterusername)}") - elif global_args.getArgs().promo is False: + elif read_args.retriveArgs().promo is False: filterusername = list( filter(lambda x: x.lowest_promo_claim is None, filterusername) ) log.debug(f"Promo Flag No Count: {len(filterusername)}") - log.debug(f"All Promo Flag: {global_args.getArgs().all_promo}") - if global_args.getArgs().all_promo: + log.debug(f"All Promo Flag: {read_args.retriveArgs().all_promo}") + if read_args.retriveArgs().all_promo: filterusername = list( filter(lambda x: x.lowest_promo_all is not None, filterusername) ) log.debug(f"All Promo Flag Yes Count: {len(filterusername)}") - elif global_args.getArgs().all_promo is False: + elif read_args.retriveArgs().all_promo is False: filterusername = list( filter(lambda x: x.lowest_promo_all is None, filterusername) ) log.debug(f"All Promo Flag No Count: {len(filterusername)}") - log.debug(f"Last Seen Flag: {global_args.getArgs().last_seen}") - if global_args.getArgs().last_seen: + log.debug(f"Last Seen Flag: {read_args.retriveArgs().last_seen}") + if read_args.retriveArgs().last_seen: filterusername = list(filter(lambda x: x.last_seen is not None, filterusername)) log.debug(f"Last Seen Flag Yes Count: {len(filterusername)}") - elif global_args.getArgs().last_seen is False: + elif read_args.retriveArgs().last_seen is False: filterusername = list(filter(lambda x: x.last_seen is None, filterusername)) log.debug(f"Last Seen Flag No Count: {len(filterusername)}") - log.debug(f"Last Seen Flag: {global_args.getArgs().last_seen}") - if global_args.getArgs().free_trial: + log.debug(f"Last Seen Flag: {read_args.retriveArgs().last_seen}") + if read_args.retriveArgs().free_trial: filterusername = list( filter( lambda x: ( @@ -52,7 +52,7 @@ def promoFilterHelper(filterusername): ) log.debug(f"Free Trial Flag Yes Count: {len(filterusername)}") - elif global_args.getArgs().free_trial is False: + elif read_args.retriveArgs().free_trial is False: filterusername = list( filter( lambda x: (x.final_current_price > 0 or x.final_regular_price == 0), diff --git a/ofscraper/filters/models/other.py b/ofscraper/filters/models/other.py index 5f9a4b817..dc7c92321 100644 --- a/ofscraper/filters/models/other.py +++ b/ofscraper/filters/models/other.py @@ -4,16 +4,16 @@ import logging -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args def otherFilters(filterusername): log = logging.getLogger("shared") - log.debug(f"Excluded usernames: {global_args.getArgs().excluded_username}") - if len(global_args.getArgs().excluded_username) > 0: + log.debug(f"Excluded usernames: {read_args.retriveArgs().excluded_username}") + if len(read_args.retriveArgs().excluded_username) > 0: filterusername = list( filter( - lambda x: x.name not in global_args.getArgs().excluded_username, + lambda x: x.name not in read_args.retriveArgs().excluded_username, filterusername, ) ) diff --git a/ofscraper/filters/models/price.py b/ofscraper/filters/models/price.py index cbdfcd829..cff061e40 100644 --- a/ofscraper/filters/models/price.py +++ b/ofscraper/filters/models/price.py @@ -1,13 +1,13 @@ import logging -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args def pricePaidFreeFilterHelper(filterusername): log = logging.getLogger("shared") - log.debug(f"Current Price Filter: {global_args.getArgs().current_price}") - if global_args.getArgs().current_price == "paid": + log.debug(f"Current Price Filter: {read_args.retriveArgs().current_price}") + if read_args.retriveArgs().current_price == "paid": filterusername = list( filter( lambda x: x.final_current_price > 0, @@ -15,7 +15,7 @@ def pricePaidFreeFilterHelper(filterusername): ) ) log.debug(f"currently paid filter username count: {len(filterusername)}") - elif global_args.getArgs().current_price == "free": + elif read_args.retriveArgs().current_price == "free": filterusername = list( filter( lambda x: x.final_current_price == 0, @@ -23,8 +23,8 @@ def pricePaidFreeFilterHelper(filterusername): ) ) log.debug(f"currently free filter username count: {len(filterusername)}") - log.debug(f"Account Renewal Price Filter: {global_args.getArgs().renewal_price}") - if global_args.getArgs().renewal_price == "paid": + log.debug(f"Account Renewal Price Filter: {read_args.retriveArgs().renewal_price}") + if read_args.retriveArgs().renewal_price == "paid": filterusername = list( filter( lambda x: x.final_renewal_price > 0, @@ -33,7 +33,7 @@ def pricePaidFreeFilterHelper(filterusername): ) log.debug(f"paid renewal filter username count: {len(filterusername)}") - elif global_args.getArgs().renewal_price == "free": + elif read_args.retriveArgs().renewal_price == "free": filterusername = list( filter( lambda x: x.final_renewal_price == 0, @@ -42,17 +42,17 @@ def pricePaidFreeFilterHelper(filterusername): ) log.debug(f"free renewal filter username count: {len(filterusername)}") - log.debug(f"Regular Price Filter: {global_args.getArgs().regular_price}") - if global_args.getArgs().regular_price == "paid": + log.debug(f"Regular Price Filter: {read_args.retriveArgs().regular_price}") + if read_args.retriveArgs().regular_price == "paid": filterusername = list(filter(lambda x: x.final_regular_price, filterusername)) log.debug(f"paid regular price filter username count: {len(filterusername)}") - elif global_args.getArgs().regular_price == "free": + elif read_args.retriveArgs().regular_price == "free": filterusername = list( filter(lambda x: x.final_regular_price == 0, filterusername) ) log.debug(f"free regular price filter username count: {len(filterusername)}") - log.debug(f"Promo Price Filter: {global_args.getArgs().promo_price}") - if global_args.getArgs().promo_price == "paid": + log.debug(f"Promo Price Filter: {read_args.retriveArgs().promo_price}") + if read_args.retriveArgs().promo_price == "paid": filterusername = list( filter( lambda x: x.final_promo_price > 0, @@ -61,7 +61,7 @@ def pricePaidFreeFilterHelper(filterusername): ) log.debug(f"paid promo filter username count: {len(filterusername)}") - elif global_args.getArgs().promo_price == "free": + elif read_args.retriveArgs().promo_price == "free": filterusername = list( filter( lambda x: x.final_promo_price == 0, @@ -76,83 +76,85 @@ def pricePaidFreeFilterHelper(filterusername): def priceMinMaxFilters(filterusername): log = logging.getLogger("shared") - log.debug(f"Promo Price min Filter: {global_args.getArgs().promo_price_min}") - if global_args.getArgs().promo_price_min: + log.debug(f"Promo Price min Filter: {read_args.retriveArgs().promo_price_min}") + if read_args.retriveArgs().promo_price_min: filterusername = list( filter( - lambda x: x.final_promo_price >= global_args.getArgs().promo_price_min, + lambda x: x.final_promo_price + >= read_args.retriveArgs().promo_price_min, filterusername, ) ) log.debug(f"currently promo min filter: {len(filterusername)}") - log.debug(f"Promo Price max Filter: {global_args.getArgs().promo_price_max}") - if global_args.getArgs().promo_price_max: + log.debug(f"Promo Price max Filter: {read_args.retriveArgs().promo_price_max}") + if read_args.retriveArgs().promo_price_max: filterusername = list( filter( - lambda x: x.final_promo_price <= global_args.getArgs().promo_price_max, + lambda x: x.final_promo_price + <= read_args.retriveArgs().promo_price_max, filterusername, ) ) log.debug(f"currently promo max filter: {len(filterusername)}") - log.debug(f"Regular Price min Filter: {global_args.getArgs().regular_price_min}") - if global_args.getArgs().regular_price_min: + log.debug(f"Regular Price min Filter: {read_args.retriveArgs().regular_price_min}") + if read_args.retriveArgs().regular_price_min: filterusername = list( filter( lambda x: x.final_regular_price - >= global_args.getArgs().regular_price_min, + >= read_args.retriveArgs().regular_price_min, filterusername, ) ) log.debug(f"currently regular min filter: {len(filterusername)}") - log.debug(f"Regular Price max Filter: {global_args.getArgs().regular_price_max}") - if global_args.getArgs().regular_price_max: + log.debug(f"Regular Price max Filter: {read_args.retriveArgs().regular_price_max}") + if read_args.retriveArgs().regular_price_max: filterusername = list( filter( lambda x: x.final_regular_price - <= global_args.getArgs().regular_price_max, + <= read_args.retriveArgs().regular_price_max, filterusername, ) ) log.debug(f"currently regular max filter: {len(filterusername)}") - log.debug(f"Renewal Price min Filter: {global_args.getArgs().renewal_price_min}") - if global_args.getArgs().renewal_price_min: + log.debug(f"Renewal Price min Filter: {read_args.retriveArgs().renewal_price_min}") + if read_args.retriveArgs().renewal_price_min: filterusername = list( filter( lambda x: x.final_renewal_price - >= global_args.getArgs().renewal_price_min, + >= read_args.retriveArgs().renewal_price_min, filterusername, ) ) log.debug(f"currently renewal min filter: {len(filterusername)}") - log.debug(f"Renewal Price max Filter: {global_args.getArgs().renewal_price_max}") - if global_args.getArgs().renewal_price_max: + log.debug(f"Renewal Price max Filter: {read_args.retriveArgs().renewal_price_max}") + if read_args.retriveArgs().renewal_price_max: filterusername = list( filter( lambda x: x.final_renewal_price - <= global_args.getArgs().renewal_price_max, + <= read_args.retriveArgs().renewal_price_max, filterusername, ) ) log.debug(f"currently renewal max filter: {len(filterusername)}") - log.debug(f"Current Price min Filter: {global_args.getArgs().current_price_min}") - if global_args.getArgs().current_price_min: + log.debug(f"Current Price min Filter: {read_args.retriveArgs().current_price_min}") + if read_args.retriveArgs().current_price_min: filterusername = list( filter( lambda x: x.final_current_price - >= global_args.getArgs().current_price_min, + >= read_args.retriveArgs().current_price_min, filterusername, ) ) log.debug(f"currently current min filter: {len(filterusername)}") - log.debug(f"Current Price max Filter: {global_args.getArgs().current_price_max}") - if global_args.getArgs().current_price_max: + log.debug(f"Current Price max Filter: {read_args.retriveArgs().current_price_max}") + if read_args.retriveArgs().current_price_max: filterusername = list( filter( lambda x: x.final_current_price - <= global_args.getArgs().current_price_max, + <= read_args.retriveArgs().current_price_max, filterusername, ) ) diff --git a/ofscraper/filters/models/selector.py b/ofscraper/filters/models/selector.py index 61f366a0f..9a8b39002 100644 --- a/ofscraper/filters/models/selector.py +++ b/ofscraper/filters/models/selector.py @@ -9,7 +9,8 @@ import ofscraper.filters.models.sort as sort import ofscraper.filters.models.subtype as subtype import ofscraper.prompts.prompts as prompts -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args +import ofscraper.utils.args.write as write_args import ofscraper.utils.constants as constants ALL_SUBS = None @@ -30,13 +31,13 @@ def getselected_usernames(rescan=False, reset=False): prompt = prompts.reset_username_prompt() if prompt == "Selection": PARSED_SUBS = None - global_args.getArgs().username = None - args_.changeargs(args) + read_args.retriveArgs().username = None + write_args.setArgs(args) if prompt == "No": rescan = False if rescan is True: PARSED_SUBS = None - if not PARSED_SUBS or not global_args.getArgs().username: + if not PARSED_SUBS or not read_args.retriveArgs().username: all_subs_helper() parsed_subscriptions_helper() return PARSED_SUBS @@ -60,16 +61,16 @@ def parsed_subscriptions_helper(force=False): global ALL_SUBS global PARSED_SUBS global args - args = global_args.getArgs() - if not global_args.getArgs().username: + args = read_args.retriveArgs() + if not read_args.retriveArgs().username: selectedusers = retriver.get_model(filterNSort((ALL_SUBS))) - global_args.getArgs().username = list(map(lambda x: x.name, selectedusers)) + read_args.retriveArgs().username = list(map(lambda x: x.name, selectedusers)) PARSED_SUBS = selectedusers - args_.changeargs(args) - elif "ALL" in global_args.getArgs().username: + write_args.setArgs(args) + elif "ALL" in read_args.retriveArgs().username: PARSED_SUBS = filterNSort(ALL_SUBS) - elif global_args.getArgs().username: - usernameset = set(global_args.getArgs().username) + elif read_args.retriveArgs().username: + usernameset = set(read_args.retriveArgs().username) PARSED_SUBS = list(filter(lambda x: x.name in usernameset, ALL_SUBS)) return PARSED_SUBS @@ -106,12 +107,12 @@ def filterNSort(usernames): f"""You have filtered the user list to zero Change the filter settings to continue -Sub Status: {global_args.getArgs().sub_status or 'No Filter'} -Renewal Status: {global_args.getArgs().renewal or 'No Filter'} -Promo Price Filter: {global_args.getArgs().promo_price or 'No Filter'} -Current Price Filter: {global_args.getArgs().current_price or 'No Filter'} -Current Price Filter: {global_args.getArgs().current_price or 'No Filter'} -Renewal Price Filter: {global_args.getArgs().renewal_price or 'No Filter'} +Sub Status: {read_args.retriveArgs().sub_status or 'No Filter'} +Renewal Status: {read_args.retriveArgs().renewal or 'No Filter'} +Promo Price Filter: {read_args.retriveArgs().promo_price or 'No Filter'} +Current Price Filter: {read_args.retriveArgs().current_price or 'No Filter'} +Current Price Filter: {read_args.retriveArgs().current_price or 'No Filter'} +Renewal Price Filter: {read_args.retriveArgs().renewal_price or 'No Filter'} """ ) diff --git a/ofscraper/filters/models/sort.py b/ofscraper/filters/models/sort.py index 69b9651c2..244738a14 100644 --- a/ofscraper/filters/models/sort.py +++ b/ofscraper/filters/models/sort.py @@ -1,9 +1,9 @@ -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args def sort_models_helper(models): - sort = global_args.getArgs().sort - reverse = global_args.getArgs().desc + sort = read_args.retriveArgs().sort + reverse = read_args.retriveArgs().desc if sort == "name": return sorted(models, reverse=reverse, key=lambda x: x.name) diff --git a/ofscraper/filters/models/subtype.py b/ofscraper/filters/models/subtype.py index 1efaa558d..c9b26ebdd 100644 --- a/ofscraper/filters/models/subtype.py +++ b/ofscraper/filters/models/subtype.py @@ -1,23 +1,23 @@ import logging -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args def subType(filterusername): log = logging.getLogger("shared") - log.debug(f"Renewal: {global_args.getArgs().renewal}") - if global_args.getArgs().renewal: + log.debug(f"Renewal: {read_args.retriveArgs().renewal}") + if read_args.retriveArgs().renewal: filterusername = list(filter(lambda x: x.renewed, filterusername)) log.debug(f"active renewal filter username count: {len(filterusername)}") - elif global_args.getArgs().renewal is False: + elif read_args.retriveArgs().renewal is False: filterusername = list(filter(lambda x: not x.renewed, filterusername)) log.debug(f"disabled renewal filter username counta: {len(filterusername)}") - log.debug(f"Sub Status: {global_args.getArgs().sub_status}") - if global_args.getArgs().sub_status: + log.debug(f"Sub Status: {read_args.retriveArgs().sub_status}") + if read_args.retriveArgs().sub_status: filterusername = list(filter(lambda x: x.active, filterusername)) log.debug(f"active subscribtion filter username count: {len(filterusername)}") - elif global_args.getArgs().sub_status is False: + elif read_args.retriveArgs().sub_status is False: filterusername = list(filter(lambda x: not x.active, filterusername)) log.debug(f"expired subscribtion filter username count: {len(filterusername)}") return filterusername diff --git a/ofscraper/prompts/model_helpers.py b/ofscraper/prompts/model_helpers.py index 98b79e1d3..88b831ba1 100644 --- a/ofscraper/prompts/model_helpers.py +++ b/ofscraper/prompts/model_helpers.py @@ -3,7 +3,7 @@ import arrow from InquirerPy.base import Choice -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args def model_selectorHelper(count, x): @@ -15,8 +15,8 @@ def model_selectorHelper(count, x): def renewHelper(x): if ( - global_args.getArgs().sort != "expired" - and global_args.getArgs().renewal is None + read_args.retriveArgs().sort != "expired" + and read_args.retriveArgs().renewal is None ): return "" return ( @@ -34,10 +34,10 @@ def generalDated(value): def lastSeenHelper(x): if ( - global_args.getArgs().sort != "last-seen" - and not global_args.getArgs().last_seen - and not global_args.getArgs().last_seen_after - and not global_args.getArgs().last_seen_before + read_args.retriveArgs().sort != "last-seen" + and not read_args.retriveArgs().last_seen + and not read_args.retriveArgs().last_seen_after + and not read_args.retriveArgs().last_seen_before ): return "" return ( @@ -50,35 +50,35 @@ def lastSeenHelper(x): def getPriceHelper(x): value = None value2 = None - if global_args.getArgs().sort in { + if read_args.retriveArgs().sort in { "current-price", "renewal-price", "regular-price", "promo-price", }: - value = re.sub("-", "_", global_args.getArgs().sort).replace("-", "_") + value = re.sub("-", "_", read_args.retriveArgs().sort).replace("-", "_") if ( - global_args.getArgs().promo_price - or global_args.getArgs().promo_price_min - or global_args.getArgs().promo_price_max + read_args.retriveArgs().promo_price + or read_args.retriveArgs().promo_price_min + or read_args.retriveArgs().promo_price_max ): value2 = "promo_price" elif ( - global_args.getArgs().regular_price - or global_args.getArgs().regular_price_min - or global_args.getArgs().regular_price_max + read_args.retriveArgs().regular_price + or read_args.retriveArgs().regular_price_min + or read_args.retriveArgs().regular_price_max ): value2 = "regular_price" elif ( - global_args.getArgs().renewal_price - or global_args.getArgs().renewal_price_min - or global_args.getArgs().renewal_price_max + read_args.retriveArgs().renewal_price + or read_args.retriveArgs().renewal_price_min + or read_args.retriveArgs().renewal_price_max ): value2 = "renewal_price" elif ( - global_args.getArgs().current_price - or global_args.getArgs().current_price_min - or global_args.getArgs().current_price_max + read_args.retriveArgs().current_price + or read_args.retriveArgs().current_price_min + or read_args.retriveArgs().current_price_max ): value2 = "current_price" final_value = value or value2 or "current_price" diff --git a/ofscraper/prompts/prompt_validators.py b/ofscraper/prompts/prompt_validators.py index 561a5bbbd..0dc952391 100644 --- a/ofscraper/prompts/prompt_validators.py +++ b/ofscraper/prompts/prompt_validators.py @@ -10,7 +10,7 @@ from prompt_toolkit.validation import ValidationError, Validator import ofscraper.classes.placeholder as placeholders -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args import ofscraper.utils.paths.check as paths_check import ofscraper.utils.profiles.data as profiles_data import ofscraper.utils.profiles.tools as profiles_tools @@ -292,7 +292,7 @@ def callable(x): def like_area_validator_posts(): def callable(x): - args = global_args.getArgs() + args = read_args.retriveArgs() if not "like" in args.action and not "unlike" in args.action: return True elif len(args.like_area) > 0: diff --git a/ofscraper/prompts/prompts.py b/ofscraper/prompts/prompts.py index 324fa5ee8..bc7880375 100644 --- a/ofscraper/prompts/prompts.py +++ b/ofscraper/prompts/prompts.py @@ -25,7 +25,8 @@ import ofscraper.prompts.prompt_strings as prompt_strings import ofscraper.prompts.prompt_validators as prompt_validators import ofscraper.prompts.promptConvert as promptClasses -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args +import ofscraper.utils.args.write as write_args import ofscraper.utils.cache as cache import ofscraper.utils.config.data as data import ofscraper.utils.config.schema as schema @@ -52,16 +53,16 @@ def action_prompt() -> int: answer = promptClasses.getChecklistSelection( message="What would you like to do?", choices=[*action_prompt_choices] ) - args = global_args.getArgs() + args = read_args.retriveArgs() action = constants.getattr("ActionPromptChoices")[answer] if action == "Main Menu": return action args.action = action - args_.changeargs(args) + write_args.setArgs(args) def areas_prompt() -> list: - args = global_args.getArgs() + args = read_args.retriveArgs() name = "value" message = ( "Which area(s) would you do you want to download and like" @@ -1024,10 +1025,10 @@ def model_selector(models) -> bool: ) def funct(prompt): - oldargs = copy.deepcopy(vars(global_args.getArgs())) + oldargs = copy.deepcopy(vars(read_args.retriveArgs())) userselector.setfilter() userselector.setsort() - if oldargs != vars(global_args.getArgs()): + if oldargs != vars(read_args.retriveArgs()): nonlocal models models = userselector.filterNSort(userselector.ALL_SUBS) choices = list( @@ -1289,7 +1290,7 @@ def decide_sort_prompt(): { "type": "list", "name": "input", - "message": f"Change the Order or the Criteria for how the model list is sorted\nCurrent setting are {global_args.getArgs().sort.capitalize()} in {'Ascending' if not global_args.getArgs().desc else 'Descending'} order", + "message": f"Change the Order or the Criteria for how the model list is sorted\nCurrent setting are {read_args.retriveArgs().sort.capitalize()} in {'Ascending' if not read_args.retriveArgs().desc else 'Descending'} order", "default": "No", "choices": ["Yes", "No"], } diff --git a/ofscraper/runner/exit.py b/ofscraper/runner/exit.py new file mode 100644 index 000000000..8e6501c12 --- /dev/null +++ b/ofscraper/runner/exit.py @@ -0,0 +1,27 @@ +from diskcache import Cache + +import ofscraper.utils.config.data as data +import ofscraper.utils.logger as logger +import ofscraper.utils.manager as manager +import ofscraper.utils.paths.paths as paths + + +def shutdown(): + logger.gracefulClose() + manager.shutdown() + + +def forcedShutDown(): + logger.forcedClose() + manager.shutdown() + + +def closeCache(): + try: + cache = Cache( + paths.getcachepath(), + disk=data.get_cache_mode(), + ) + cache.close() + except Exception as E: + raise E diff --git a/ofscraper/runner/load.py b/ofscraper/runner/load.py new file mode 100644 index 000000000..183f603bb --- /dev/null +++ b/ofscraper/runner/load.py @@ -0,0 +1,61 @@ +import logging +import sys +import traceback + +import ofscraper.runner.run as run +import ofscraper.utils.args.read as read_args +import ofscraper.utils.config.config as config_ +import ofscraper.utils.dates as dates +import ofscraper.utils.logger as logger +import ofscraper.utils.paths.manage as paths_manage +import ofscraper.utils.system.system as system + + +def main(): + try: + systemSet() + args_loader() + setdate() + setLogger() + readConfig() + make_folder() + run.main() + except SystemExit: + raise Exception(f"Arguments {sys.argv}") + except Exception as E: + try: + logging.getLogger("shared").debug(traceback.format_exc()) + logging.getLogger("shared").debug(E) + except: + print(E) + print(traceback.format_exc()) + + +def args_loader(): + read_args.retriveArgs() + + +def setdate(): + dates.setLogDate() + + +def setLogger(): + logger.init_values() + logger.get_shared_logger() + logger.discord_warning() + logger.start_stdout_logthread() + logger.start_other_helper() + + +def systemSet(): + system.setName() + system.set_mulitproc_start_type() + system.set_eventloop() + + +def readConfig(): + config_.read_config() + + +def make_folder(): + paths_manage.make_folders() diff --git a/ofscraper/runner/run.py b/ofscraper/runner/run.py new file mode 100644 index 000000000..901f02edc --- /dev/null +++ b/ofscraper/runner/run.py @@ -0,0 +1,54 @@ +import logging +import time +import traceback + +import ofscraper.commands.picker as picker +import ofscraper.runner.exit as exit_manager +import ofscraper.runner.startvals as startvals +import ofscraper.utils.args.read as read_args +import ofscraper.utils.console as console +import ofscraper.utils.context.exit as exit_context +import ofscraper.utils.system.system as system + + +def main(): + try: + main_helper() + except KeyboardInterrupt as E: + console.get_shared_console().print("handling force closing of script") + try: + with exit_context.DelayedKeyboardInterrupt(): + exit_manager.forcedShutDown() + exit_manager.closeCache() + except KeyboardInterrupt as E: + with exit_context.DelayedKeyboardInterrupt(): + exit_manager.forcedShutDown() + raise E + except Exception as E: + logging.getLogger("shared").debug(traceback.format_exc()) + logging.getLogger("shared").debug(E) + try: + with exit_context.DelayedKeyboardInterrupt(): + exit_manager.forcedShutDown() + exit_manager.closeCache() + except KeyboardInterrupt as E: + with exit_context.DelayedKeyboardInterrupt(): + exit_manager.forcedShutDown() + raise E + + +def main_helper(): + if vars(read_args.retriveArgs()).get("help"): + return + initLogs() + time.sleep(3) + picker.pick() + exit_manager.shutdown() + + +def initLogs(): + if len(system.get_dupe_ofscraper()) > 0: + console.get_shared_console().print( + "[bold yellow]Warning another OF-Scraper instance was detected[bold yellow]\n\n\n" + ) + startvals.printStartValues() diff --git a/ofscraper/start.py b/ofscraper/runner/start.py similarity index 80% rename from ofscraper/start.py rename to ofscraper/runner/start.py index ff87e5913..bdbf57b1e 100644 --- a/ofscraper/start.py +++ b/ofscraper/runner/start.py @@ -8,8 +8,7 @@ def main(): from diskcache import Cache import ofscraper.commands.picker as picker - import ofscraper.utils.args.parse as args_parse - import ofscraper.utils.config.config as config_ + import ofscraper.utils.args.read as read_args import ofscraper.utils.config.data as data import ofscraper.utils.console as console import ofscraper.utils.context.exit as exit @@ -20,31 +19,20 @@ def main(): import ofscraper.utils.startvals as startvals import ofscraper.utils.system.system as system - system.setName() - system.set_mulitproc_start_type() - config_.read_config() if len(system.get_dupe_ofscraper()) > 0: console.get_shared_console().print( "[bold yellow]Warning another OF-Scraper instance was detected[bold yellow]\n\n\n" ) - logger.init_values() - system.set_eventloop() - logger.get_shared_logger() startvals.printStartValues() - logger.discord_warning() - args = args_parse.parse_args() + args = read_args.retriveArgs() if vars(args).get("help"): return - logger.start_stdout_logthread() - logger.start_other_helper() # allow background processes to start time.sleep(3) - paths_manage.make_folders() picker.pick() - logger.gracefulClose() - manager.shutdown() + except SystemExit: print(f"Arguments {sys.argv}") except KeyboardInterrupt as E: @@ -70,8 +58,8 @@ def main(): manager.shutdown() raise E except Exception as E: - logging.getLogger("shared").traceback_(traceback.format_exc()) - logging.getLogger("shared").traceback_(E) + logging.getLogger("shared").debug(traceback.format_exc()) + logging.getLogger("shared").debug(E) try: with exit.DelayedKeyboardInterrupt(): logger.forcedClose() diff --git a/ofscraper/runner/startvals.py b/ofscraper/runner/startvals.py new file mode 100644 index 000000000..34d20ea67 --- /dev/null +++ b/ofscraper/runner/startvals.py @@ -0,0 +1,34 @@ +import logging +import platform +import ssl +import sys + +import certifi + +import ofscraper.utils.args.read as read_args +import ofscraper.utils.config.file as config_file +import ofscraper.utils.logger as logger +import ofscraper.utils.paths.common as common_paths +import ofscraper.utils.system.system as system + + +def printStartValues(): + args = read_args.retriveArgs() + log = logging.getLogger("shared") + logger.updateSenstiveDict(f"/{common_paths.get_username()}/", "/your_username/") + logger.updateSenstiveDict( + f"\\{common_paths.get_username()}\\", "\\\\your_username\\\\" + ) + + # print info + log.debug(args) + log.debug(sys.argv[1:]) if len(sys.argv) > 1 else None + log.debug(platform.platform()) + log.debug(config_file.open_config()) + log.info(f"config path: {str(common_paths.get_config_path())}") + log.info(f"profile path: {str(common_paths.get_profile_path())}") + log.info(f"log folder: {str(common_paths.get_config_home()/'logging')}") + log.debug(f"ssl {ssl.get_default_verify_paths()}") + log.debug(f"python version {platform. python_version()}") + log.debug(f"certifi {certifi.where()}") + log.debug(f"number of threads available on system {system.getcpu_count()}") diff --git a/ofscraper/utils/actions.py b/ofscraper/utils/actions.py index 78d058178..ac0a55f3e 100644 --- a/ofscraper/utils/actions.py +++ b/ofscraper/utils/actions.py @@ -8,12 +8,14 @@ \/ \/ \/ \/ """ import ofscraper.prompts.prompts as prompts -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.areas as areas +import ofscraper.utils.args.read as read_args +import ofscraper.utils.args.write as write_args import ofscraper.utils.system.free as free def reset_download(): - args = global_args.getArgs() + args = read_args.retriveArgs() if bool(args.download_area) and prompts.reset_download_areas_prompt() == "Yes": args.scrape_paid = None @@ -21,20 +23,20 @@ def reset_download(): def reset_like(): - args = global_args.getArgs() + args = read_args.retriveArgs() if bool(args.like_area) and prompts.reset_like_areas_prompt() == "Yes": args.like_area = {} @free.space_checker def select_areas(action=None, reset=False): - args = global_args.getArgs() + args = read_args.retriveArgs() action = action or args.action if "download" in action and reset: reset_download() elif ("like" or "unlike") in action and reset: reset_like() - args_.changeargs(args) + write_args.setArgs(args) set_post_area(action) set_download_area(action) set_scrape_paid(action) @@ -43,55 +45,55 @@ def select_areas(action=None, reset=False): def remove_post_area(): - args = global_args.getArgs() + args = read_args.retriveArgs() args.posts = {} - args_.changeargs(args) + write_args.setArgs(args) @free.space_checker # set post for primarily for download-area, secondary for like/unlike def set_post_area(action=None): - args = global_args.getArgs() + args = read_args.retriveArgs() action = action or args.action or {} if "download" not in action: return - elif len(args_.get_download_area()) > 0: + elif len(areas.get_download_area()) > 0: return elif len(args.posts) > 0: return args.posts = prompts.areas_prompt() - args_.changeargs(args) + write_args.setArgs(args) # set download_area based on posts def set_download_area(action=None): - args = global_args.getArgs() + args = read_args.retriveArgs() action = action or args.action or {} if "download" not in action: return - args.download_area = args_.get_download_area() + args.download_area = areas.get_download_area() def set_scrape_paid(action=None): - args = global_args.getArgs() + args = read_args.retriveArgs() action = action or args.action or {} if "download" not in action: return args.scrape_paid = ( prompts.scrape_paid_prompt() if args.scrape_paid != None else args.scrape_paid ) - args_.changeargs(args) + write_args.setArgs(args) # set like area based primarly on posts,secondary on from prompt def set_like_area(action=None): - args = global_args.getArgs() + args = read_args.retriveArgs() action = action or args.action or {} if "like" not in action and "unlike" not in action: return args.like_area = ( - args_.get_like_area() - if len(args_.get_like_area()) > 0 + areas.get_like_area() + if len(areas.get_like_area()) > 0 else prompts.like_areas_prompt() ) - args_.changeargs(args) + write_args.setArgs(args) diff --git a/ofscraper/utils/args/areas.py b/ofscraper/utils/args/areas.py index e37ea92cd..5f6d01c56 100644 --- a/ofscraper/utils/args/areas.py +++ b/ofscraper/utils/args/areas.py @@ -1,4 +1,4 @@ -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args def get_like_area(): @@ -9,10 +9,10 @@ def get_like_area(): "Pinned", "Labels", ] - if len(global_args.getArgs().like_area) == 0: - post = set(global_args.getArgs().posts) + if len(read_args.retriveArgs().like_area) == 0: + post = set(read_args.retriveArgs().posts) else: - post = set(global_args.getArgs().like_area) + post = set(read_args.retriveArgs().like_area) if "All" in post: post.update(set(all_choices)) return list( @@ -39,10 +39,10 @@ def get_download_area(): "Profile", "Labels", ] - if len(global_args.getArgs().download_area) == 0: - post = set(global_args.getArgs().posts) + if len(read_args.retriveArgs().download_area) == 0: + post = set(read_args.retriveArgs().posts) else: - post = set(global_args.getArgs().download_area) + post = set(read_args.retriveArgs().download_area) if "All" in post: post.update(set(all_choices)) return list( diff --git a/ofscraper/utils/args/args.py b/ofscraper/utils/args/args.py deleted file mode 100644 index 6ab58b0ea..000000000 --- a/ofscraper/utils/args/args.py +++ /dev/null @@ -1,14 +0,0 @@ -import argparse -import pathlib -import re -import sys - -import arrow -from humanfriendly import parse_size - -import ofscraper.utils.config.data as config_data -import ofscraper.utils.system.system as system -from ofscraper.__version__ import __version__ -from ofscraper.const.constants import KEY_OPTIONS, OFSCRAPER_RESERVED_LIST - -args = None diff --git a/ofscraper/utils/args/globals.py b/ofscraper/utils/args/globals.py index a80805ab5..1022b94c6 100644 --- a/ofscraper/utils/args/globals.py +++ b/ofscraper/utils/args/globals.py @@ -1,11 +1 @@ args = None - - -def setArgs(changed): - global args - args = changed - - -def getArgs(): - global args - return args diff --git a/ofscraper/utils/args/helpers.py b/ofscraper/utils/args/helpers.py index c8bab1113..a659871d3 100644 --- a/ofscraper/utils/args/helpers.py +++ b/ofscraper/utils/args/helpers.py @@ -1,50 +1,11 @@ import argparse import pathlib import re -import sys import arrow from humanfriendly import parse_size -import ofscraper.utils.config.data as config_data -import ofscraper.utils.system.system as system from ofscraper.__version__ import __version__ -from ofscraper.const.constants import KEY_OPTIONS, OFSCRAPER_RESERVED_LIST - - -def globalDataHelper(): - global args - now = arrow.now() - args.log_dateformat = getDateHelper(now) - args.date_now = getDateNowHelper(now) - return args - - -def resetGlobalDateHelper(): - clearDate() - globalDataHelper() - - -def clearDate(): - global args - args.date_now = None - args.log_dateformat = None - - -def getDateNowHelper(now): - if not vars(args).get("date_now"): - return now - return args.date_now - - -def getDateHelper(now): - if not vars(args).get("log_dateformat"): - return ( - now.format("YYYY-MM-DD") - if config_data.get_appendlog() - else f'{now.format("YYYY-MM-DD_hh.mm.ss")}' - ) - return args.log_dateformat def check_strhelper(x): diff --git a/ofscraper/utils/args/parse.py b/ofscraper/utils/args/parse.py index 401790e1a..1ece232ec 100644 --- a/ofscraper/utils/args/parse.py +++ b/ofscraper/utils/args/parse.py @@ -4,8 +4,8 @@ from humanfriendly import parse_size -import ofscraper.utils.args.globals as global_args import ofscraper.utils.args.helpers as helpers +import ofscraper.utils.args.write as write_args import ofscraper.utils.system.system as system from ofscraper.__version__ import __version__ from ofscraper.const.constants import KEY_OPTIONS, OFSCRAPER_RESERVED_LIST @@ -865,7 +865,6 @@ def parse_args(input=None): args.excluded_username = set(args.excluded_username or []) args.label = set(args.label) if args.label else args.label args.black_list = set(list(map(lambda x: x.lower(), args.black_list))) - args = helpers.globalDataHelper() if len(args.user_list) == 0: args.user_list = {OFSCRAPER_RESERVED_LIST} @@ -886,5 +885,5 @@ def parse_args(input=None): raise argparse.ArgumentTypeError( "error: argument missing --url or --file must be specified )" ) - global_args.setArgs(args) + write_args.setArgs(args) return args diff --git a/ofscraper/utils/args/read.py b/ofscraper/utils/args/read.py new file mode 100644 index 000000000..511554908 --- /dev/null +++ b/ofscraper/utils/args/read.py @@ -0,0 +1,6 @@ +import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.parse as parse_args + + +def retriveArgs(): + return global_args.args or parse_args.parse_args() diff --git a/ofscraper/utils/args/write.py b/ofscraper/utils/args/write.py new file mode 100644 index 000000000..715ccbda0 --- /dev/null +++ b/ofscraper/utils/args/write.py @@ -0,0 +1,5 @@ +import ofscraper.utils.args.globals as global_args + + +def setArgs(changed): + global_args.args = changed diff --git a/ofscraper/utils/auth.py b/ofscraper/utils/auth.py index 7e10221f9..e2ec47fff 100644 --- a/ofscraper/utils/auth.py +++ b/ofscraper/utils/auth.py @@ -28,7 +28,7 @@ import ofscraper.classes.sessionbuilder as sessionbuilder import ofscraper.prompts.prompts as prompts -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args import ofscraper.utils.config.data as data import ofscraper.utils.constants as constants import ofscraper.utils.paths.common as common_paths @@ -293,7 +293,7 @@ def make_request_auth(): def get_request_auth(): - if (global_args.getArgs().dynamic_rules or data.get_dynamic() or "deviint") in { + if (read_args.retriveArgs().dynamic_rules or data.get_dynamic() or "deviint") in { "deviint", "dv", "dev", diff --git a/ofscraper/utils/cache.py b/ofscraper/utils/cache.py index 0f793997c..a354cb193 100644 --- a/ofscraper/utils/cache.py +++ b/ofscraper/utils/cache.py @@ -1,6 +1,6 @@ from diskcache import Cache -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args import ofscraper.utils.config.data as data import ofscraper.utils.paths.common as common_paths @@ -8,7 +8,7 @@ def get(*args, **kwargs): - if global_args.getArgs().no_cache or data.get_cache_mode() == "disabled": + if read_args.retriveArgs().no_cache or data.get_cache_mode() == "disabled": return kwargs.get("default") global cache if cache is None: @@ -17,7 +17,7 @@ def get(*args, **kwargs): def set(*args, **kwargs): - if global_args.getArgs().no_cache or data.get_cache_mode() == "disabled": + if read_args.retriveArgs().no_cache or data.get_cache_mode() == "disabled": return global cache if cache is None: @@ -26,7 +26,7 @@ def set(*args, **kwargs): def close(*args, **kwargs): - if global_args.getArgs().no_cache or data.get_cache_mode() == "disabled": + if read_args.retriveArgs().no_cache or data.get_cache_mode() == "disabled": return None global cache if cache is None: @@ -35,7 +35,7 @@ def close(*args, **kwargs): def touch(*args, **kwargs): - if global_args.getArgs().no_cache or data.get_cache_mode() == "disabled": + if read_args.retriveArgs().no_cache or data.get_cache_mode() == "disabled": return None global cache if cache is None: diff --git a/ofscraper/utils/config/file.py b/ofscraper/utils/config/file.py index 480159429..54bb6efa0 100644 --- a/ofscraper/utils/config/file.py +++ b/ofscraper/utils/config/file.py @@ -4,8 +4,6 @@ from humanfriendly import parse_size -import ofscraper.prompts.prompts as prompts -import ofscraper.utils.binaries as binaries import ofscraper.utils.config.schema as schema import ofscraper.utils.console as console_ import ofscraper.utils.paths.common as common_paths diff --git a/ofscraper/utils/constants.py b/ofscraper/utils/constants.py index 6a0c18b88..20bb76660 100644 --- a/ofscraper/utils/constants.py +++ b/ofscraper/utils/constants.py @@ -8,8 +8,6 @@ def getattr(val): global custom #can not be overwritten cause of infinite loop - if val in { "configPath","configFile"}: - return globals()[val] if custom ==None: custom=custom_.get_custom() return (custom or {}).get(val) or globals()[val] diff --git a/ofscraper/utils/context/stdout.py b/ofscraper/utils/context/stdout.py index f21632890..b5129a795 100644 --- a/ofscraper/utils/context/stdout.py +++ b/ofscraper/utils/context/stdout.py @@ -3,13 +3,13 @@ import os import sys -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args import ofscraper.utils.constants as constants @contextlib.contextmanager def lowstdout(): - if global_args.getArgs().output in constants.getattr("SUPRESS_OUTPUTS"): + if read_args.retriveArgs().output in constants.getattr("SUPRESS_OUTPUTS"): save_stdout = sys.stdout sys.stdout = open(os.devnull, "w") yield diff --git a/ofscraper/utils/dates.py b/ofscraper/utils/dates.py index 0272b72e8..b5aab2b35 100644 --- a/ofscraper/utils/dates.py +++ b/ofscraper/utils/dates.py @@ -12,6 +12,10 @@ import arrow +import ofscraper.utils.manager as manager + +dateFormat = None + def convert_date_to_mdyhms(date: str): datetime_obj = datetime.fromisoformat(date) @@ -34,3 +38,26 @@ def convert_local_time(date: str): def get_current_time(): return arrow.get(tzinfo="UTC").to("local").float_timestamp + + +def setLogDate(dateDict=None): + global dateFormat + if not dateDict: + dateDict = { + "day": arrow.now().format("YYYY-MM-DD"), + "now": arrow.now().format("YYYY-MM-DD_hh.mm.ss"), + } + dateFormat = manager.get_manager().dict(dateDict) + + +def getLogDate(): + global dateFormat + if not dateFormat: + setLogDate() + return dateFormat + + +def resetLogdate(): + global dateFormat + dateFormat = None + setLogDate() diff --git a/ofscraper/utils/logger.py b/ofscraper/utils/logger.py index af6149f16..8ff1fa6ee 100644 --- a/ofscraper/utils/logger.py +++ b/ofscraper/utils/logger.py @@ -19,10 +19,12 @@ ) import ofscraper.classes.sessionbuilder as sessionbuilder -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args +import ofscraper.utils.args.write as write_args import ofscraper.utils.config.data as data import ofscraper.utils.console as console import ofscraper.utils.constants as constants +import ofscraper.utils.dates as dates import ofscraper.utils.manager as manager_ import ofscraper.utils.paths.common as common_paths import ofscraper.utils.system.system as system @@ -305,17 +307,17 @@ def init_stdout_logger(name=None): show_level=False, console=console.get_shared_console(), ) - sh.setLevel(getLevel(global_args.getArgs().output)) + sh.setLevel(getLevel(read_args.retriveArgs().output)) sh.setFormatter(SensitiveFormatter(format)) sh.addFilter(NoDebug()) tx = TextHandler() - tx.setLevel(getLevel(global_args.getArgs().output)) + tx.setLevel(getLevel(read_args.retriveArgs().output)) tx.setFormatter(SensitiveFormatter(format)) log.addHandler(sh) log.addHandler(tx) - if global_args.getArgs().output in {"TRACE", "DEBUG"}: - funct = DebugOnly if global_args.getArgs().output == "DEBUG" else TraceOnly + if read_args.retriveArgs().output in {"TRACE", "DEBUG"}: + funct = DebugOnly if read_args.retriveArgs().output == "DEBUG" else TraceOnly sh2 = RichHandler( rich_tracebacks=True, console=console.get_shared_console(), @@ -323,7 +325,7 @@ def init_stdout_logger(name=None): tracebacks_show_locals=True, show_time=False, ) - sh2.setLevel(global_args.getArgs().output) + sh2.setLevel(read_args.retriveArgs().output) sh2.setFormatter(SensitiveFormatter(format)) sh2.addFilter(funct()) log.addHandler(sh2) @@ -341,25 +343,25 @@ def init_other_logger(name): # # #log file # #discord cord = DiscordHandler() - cord.setLevel(getLevel(global_args.getArgs().discord)) + cord.setLevel(getLevel(read_args.retriveArgs().discord)) cord.setFormatter(SensitiveFormatter("%(message)s")) # console log.addHandler(cord) - if global_args.getArgs().log != "OFF": + if read_args.retriveArgs().log != "OFF": stream = open( common_paths.getlogpath(), encoding="utf-8", mode="a", ) fh = logging.StreamHandler(stream) - fh.setLevel(getLevel(global_args.getArgs().log)) + fh.setLevel(getLevel(read_args.retriveArgs().log)) fh.setFormatter(LogFileFormatter(format, "%Y-%m-%d %H:%M:%S")) fh.addFilter(NoDebug()) log.addHandler(fh) - if global_args.getArgs().log in {"TRACE", "DEBUG"}: - funct = DebugOnly if global_args.getArgs().output == "DEBUG" else TraceOnly + if read_args.retriveArgs().log in {"TRACE", "DEBUG"}: + funct = DebugOnly if read_args.retriveArgs().output == "DEBUG" else TraceOnly fh2 = logging.StreamHandler(stream) - fh2.setLevel(getLevel(global_args.getArgs().log)) + fh2.setLevel(getLevel(read_args.retriveArgs().log)) fh2.setFormatter(LogFileFormatter(format, "%Y-%m-%d %H:%M:%S")) fh2.addFilter(funct()) log.addHandler(fh2) @@ -368,9 +370,12 @@ def init_other_logger(name): # updates stream for main process def updateOtherLoggerStream(): - if global_args.getArgs().discord == "OFF" and global_args.getArgs().log == "OFF": + if ( + read_args.retriveArgs().discord == "OFF" + and read_args.retriveArgs().log == "OFF" + ): return - args.resetGlobalDateHelper() + dates.resetLogdate() stream = open( common_paths.getlogpath(), encoding="utf-8", @@ -505,9 +510,9 @@ def start_stdout_logthread(input_=None, name=None, count=1, event=None): # wrapper function for discord and log, check if threads/process should star def start_checker(func: abc.Callable): def inner(*args_, **kwargs): - if global_args.getArgs().discord and global_args.getArgs().discord != "OFF": + if read_args.retriveArgs().discord and read_args.retriveArgs().discord != "OFF": return func(*args_, **kwargs) - elif global_args.getArgs().log and global_args.getArgs().log != "OFF": + elif read_args.retriveArgs().log and read_args.retriveArgs().log != "OFF": return func(*args_, **kwargs) return inner @@ -531,7 +536,7 @@ def start_other_thread(input_=None, count=1, name=None, other_event=None): @start_checker def start_other_process(input_=None, count=1): def inner(args_, input_=None, count=1): - args.changeargs(args_) + write_args.setArgs(args_) input_ = input_ or otherqueue_ logger_other(input_, stop_count=count) @@ -539,7 +544,7 @@ def inner(args_, input_=None, count=1): input_ = otherqueue_ process = aioprocessing.AioProcess( target=inner, - args=(global_args.getArgs(),), + args=(read_args.retriveArgs(),), kwargs={"input_": input_, "count": count}, daemon=True, ) @@ -568,11 +573,11 @@ def get_shared_logger(main_=None, other_=None, name=None): mainhandle = QueueHandler(main_) elif hasattr(main_, "send"): mainhandle = PipeHandler(main_) - mainhandle.setLevel(getLevel(global_args.getArgs().output)) + mainhandle.setLevel(getLevel(read_args.retriveArgs().output)) # add a handler that uses the shared queue logger.addHandler(mainhandle) - discord_level = getNumber(global_args.getArgs().discord) - file_level = getNumber(global_args.getArgs().log) + discord_level = getNumber(read_args.retriveArgs().discord) + file_level = getNumber(read_args.retriveArgs().log) other_ = other_ or otherqueue_ if hasattr(main_, "get") and hasattr(main_, "put_nowait"): otherhandle = QueueHandler(other_) @@ -666,7 +671,7 @@ def closeQueue(): def discord_warning(): - if global_args.getArgs().discord == "DEBUG": + if read_args.retriveArgs().discord == "DEBUG": console.get_shared_console().print( "[bold red]Warning Discord with DEBUG is not recommended\nAs processing messages is much slower compared to other[/bold red]" ) diff --git a/ofscraper/utils/paths/common.py b/ofscraper/utils/paths/common.py index ed6042b5c..ad1d7080c 100644 --- a/ofscraper/utils/paths/common.py +++ b/ofscraper/utils/paths/common.py @@ -4,9 +4,10 @@ import arrow import ofscraper.const.constants as consts -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args import ofscraper.utils.config.data as data import ofscraper.utils.constants as constants +import ofscraper.utils.dates as dates_manager import ofscraper.utils.profiles.data as profile_data @@ -36,7 +37,7 @@ def get_config_home(): def get_config_path(): - configPath = global_args.getArgs().config + configPath = read_args.retriveArgs().config defaultPath = pathlib.Path.home() / consts.configPath / consts.configFile ofscraperHome = pathlib.Path.home() / consts.configPath @@ -58,20 +59,19 @@ def get_config_path(): def getlogpath(): - logDate = global_args.getArgs().log_dateformat path = None if not data.get_appendlog(): path = ( get_config_home() / "logging" - / f'{data.get_main_profile()}_{arrow.now().format("YYYY-MM-DD")}' - / f"ofscraper_{data.get_main_profile()}_{logDate}.log" + / f'{data.get_main_profile()}_{dates_manager.getLogDate().get("day")}' + / f'ofscraper_{data.get_main_profile()}_{dates_manager.getLogDate().get("day")}.log' ) else: path = ( get_config_home() / "logging" - / f"ofscraper_{data.get_main_profile()}_{logDate}.log" + / f'ofscraper_{data.get_main_profile()}_{dates_manager.getLogDate().get("now")}.log' ) path = pathlib.Path(path).resolve() path.parent.mkdir(parents=True, exist_ok=True) @@ -81,9 +81,9 @@ def getlogpath(): def get_profile_path(name=None): if name: return get_config_home() / name - elif not global_args.getArgs().profile: + elif not read_args.retriveArgs().profile: return get_config_home() / profile_data.get_current_config_profile() - return get_config_home() / global_args.getArgs().profile + return get_config_home() / read_args.retriveArgs().profile def get_save_location(config=None): diff --git a/ofscraper/utils/paths/paths.py b/ofscraper/utils/paths/paths.py index 146414314..da7db9d89 100644 --- a/ofscraper/utils/paths/paths.py +++ b/ofscraper/utils/paths/paths.py @@ -6,7 +6,7 @@ from contextlib import contextmanager from pathlib import Path -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args import ofscraper.utils.config.data as data import ofscraper.utils.console as console_ import ofscraper.utils.paths.common as common_paths @@ -37,7 +37,11 @@ def set_directory(path: Path): def cleanup(): - if global_args.getArgs().no_auto_resume or not data.get_part_file_clean() or False: + if ( + read_args.retriveArgs().no_auto_resume + or not data.get_part_file_clean() + or False + ): log.info("Cleaning up temp files\n\n") root = pathlib.Path(data.get_TempDir() or common_paths.get_save_location()) for file in list( diff --git a/ofscraper/utils/profiles/data.py b/ofscraper/utils/profiles/data.py index 2c5ceb759..2a569c9ae 100644 --- a/ofscraper/utils/profiles/data.py +++ b/ofscraper/utils/profiles/data.py @@ -2,7 +2,7 @@ from rich import print -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args import ofscraper.utils.config.data as data import ofscraper.utils.paths.common as common_paths @@ -42,6 +42,6 @@ def get_current_config_profile(): def get_active_profile(): - if global_args.getArgs().profile: - return global_args.getArgs().profile + if read_args.retriveArgs().profile: + return read_args.retriveArgs().profile return get_current_config_profile() diff --git a/ofscraper/utils/profiles/manage.py b/ofscraper/utils/profiles/manage.py index 689d0a942..6464861d1 100644 --- a/ofscraper/utils/profiles/manage.py +++ b/ofscraper/utils/profiles/manage.py @@ -4,7 +4,8 @@ from rich import print import ofscraper.prompts.prompts as prompts -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args +import ofscraper.utils.args.write as write_args import ofscraper.utils.config.data as data import ofscraper.utils.console as console import ofscraper.utils.constants as constants @@ -23,10 +24,10 @@ def change_profile(): tools.print_profiles() profile = prompts.get_profile_prompt(profile_data.get_profile_names()) data.update_config(constants.getattr("mainProfile"), profile) - args = global_args.getArgs() + args = read_args.retriveArgs() # remove profile argument args.profile = None - args_.changeargs(args) + write_args.setArgs(args) print(f"[green]Successfully changed profile to[/green] {profile}") @@ -72,10 +73,10 @@ def edit_profile_name(): def change_current_profile(new_profile, old_profile): - args = global_args.getArgs() + args = read_args.retriveArgs() if args.profile == old_profile: args.profile = new_profile - args_.changeargs(args) + write_args.setArgs(args) # required because name has changed if old_profile == profile_data.get_current_config_profile(): data.update_config(constants.getattr("mainProfile"), new_profile) diff --git a/ofscraper/utils/startvals.py b/ofscraper/utils/startvals.py index 452302d6a..e69de29bb 100644 --- a/ofscraper/utils/startvals.py +++ b/ofscraper/utils/startvals.py @@ -1,34 +0,0 @@ -import logging -import platform -import ssl -import sys - -import certifi - -import ofscraper.utils.args.globals as global_args -import ofscraper.utils.config.file as config_file -import ofscraper.utils.logger as logger -import ofscraper.utils.paths.common as common_paths -import ofscraper.utils.system.system as system - - -def printStartValues(): - args = global_args.getArgs() - log = logging.getLogger("shared") - logger.updateSenstiveDict(f"/{common_paths.get_username()}/", "/your_username/") - logger.updateSenstiveDict( - f"\\{common_paths.get_username()}\\", "\\\\your_username\\\\" - ) - - # print info - log.debug(args) - log.debug(sys.argv[1:]) if len(sys.argv) > 1 else None - log.debug(platform.platform()) - log.debug(config_file.open_config()) - log.info(f"config path: {str(common_paths.get_config_path())}") - log.info(f"profile path: {str(common_paths.get_profile_path())}") - log.info(f"log folder: {str(common_paths.get_config_home()/'logging')}") - log.debug(f"ssl {ssl.get_default_verify_paths()}") - log.debug(f"python version {platform. python_version()}") - log.debug(f"certifi {certifi.where()}") - log.debug(f"number of threads available on system {system.getcpu_count()}") diff --git a/ofscraper/utils/system/network.py b/ofscraper/utils/system/network.py index 142b7f727..4e5997fac 100644 --- a/ofscraper/utils/system/network.py +++ b/ofscraper/utils/system/network.py @@ -4,7 +4,7 @@ import httpx import ofscraper.classes.sessionbuilder as sessionbuilder -import ofscraper.utils.args.globals as global_args +import ofscraper.utils.args.read as read_args import ofscraper.utils.config.data as data import ofscraper.utils.console as console_ import ofscraper.utils.constants as constants @@ -16,7 +16,7 @@ def check_cdm(): console = console_.get_shared_console() log = logging.getLogger("shared") - keymode = global_args.getArgs().key_mode or data.get_key_mode() or "cdrm" + keymode = read_args.retriveArgs().key_mode or data.get_key_mode() or "cdrm" console.print(f"[yellow]Key Mode: {keymode}\n\n[/yellow]") if keymode == "manual": console.print(