allow changing list and fix issues with changing list

use setttings.py instead of config and args seperate

ofscraper/api/subscriptions/helpers.py

@@ -24,9 +24,8 @@
     wait_random,
 )
 
-import ofscraper.utils.args.read as read_args
-import ofscraper.utils.config.data as data
 import ofscraper.utils.constants as constants
+import ofscraper.utils.settings as settings
 from ofscraper.classes.semaphoreDelayed import semaphoreDelayed
 from ofscraper.utils.context.run_async import run
 
@@ -37,14 +36,14 @@
 
 
 def get_user_list_helper():
-    out = read_args.retriveArgs().user_list or data.get_default_userlist()
+    out = settings.get_userlist()
     if isinstance(out, str):
         out = out.split(",")
     return set(map(lambda x: x.lower().strip(), out))
 
 
 def get_black_list_helper():
-    out = read_args.retriveArgs().black_list or data.get_default_blacklist()
+    out = settings.get_blacklist()
     if isinstance(out, str):
         out = out.split(",")
     return set(map(lambda x: x.lower().strip(), out))

ofscraper/classes/placeholder.py

@@ -16,6 +16,7 @@
 import ofscraper.utils.paths.common as common_paths
 import ofscraper.utils.paths.paths as paths
 import ofscraper.utils.profiles.data as profile_data
+import ofscraper.utils.settings as settings
 
 log = logging.getLogger("shared")
 
@@ -295,11 +296,11 @@ def _addcount(self, ele, out):
         return out
 
     def set_final_path(self):
-        if (read_args.retriveArgs().original or data.get_truncation()) is False:
+        if settings.get_trunication is False:
             self._final_path = pathlib.Path(self.mediadir, f"{self.filename}")
         elif read_args.retriveArgs().original is False:
             self._final_path = pathlib.Path(self.mediadir, f"{self.filename}")
-        elif read_args.retriveArgs().original is True or data.get_truncation() is True:
+        elif settings.get_trunication is True:
             self._final_path = paths.truncate(
                 pathlib.Path(self.mediadir, f"{self.filename}")
             )

ofscraper/commands/scraper.py

@@ -35,8 +35,6 @@
 
 def process_selected_areas():
     log.debug(f"[bold blue] Running Action Mode [/bold blue]")
-    checkers.check_auth()
-    checkers.check_config()
     functs = process_actions.add_selected_areas()
     run.run_helper(*functs)
     while True:
@@ -56,8 +54,6 @@ def process_selected_areas():
 
 
 def daemon_process():
-    checkers.check_auth()
-    checkers.check_config()
     functs = process_actions.add_selected_areas()
     run.daemon_run_helper(*functs)
 

ofscraper/const/prompts.py

@@ -57,5 +57,6 @@
     "Filter model list based on promotion status": "promo",
     "Filter model list based on status of prices": "price",
     "Change sorting of model list": "sort",
+    "Change to a different list [May require rescan]": "list",
     "Go Back to Model Lists": "modelList",
 }

ofscraper/download/common.py

@@ -57,6 +57,7 @@
 import ofscraper.utils.constants as constants
 import ofscraper.utils.dates as dates
 import ofscraper.utils.paths.common as common_paths
+import ofscraper.utils.settings as settings
 import ofscraper.utils.system as system
 from ofscraper.classes.multiprocessprogress import MultiprocessProgress as MultiProgress
 from ofscraper.classes.semaphoreDelayed import semaphoreDelayed
@@ -305,10 +306,8 @@ async def check_forced_skip(ele, *args):
     total = sum(map(lambda x: int(x), args))
     if total == 0:
         return 0
-    file_size_limit = (
-        read_args.retriveArgs().size_max or config_data.get_filesize_limit()
-    )
-    file_size_min = read_args.retriveArgs().size_min or config_data.get_filesize_limit()
+    file_size_limit = settings.get_size_limit()
+    file_size_min = settings.get_size_min()
     if int(file_size_limit) > 0 and (int(total) > int(file_size_limit)):
         ele.mediatype = "forced_skipped"
         log.debug(f"{get_medialog(ele)} {format_size(total)} over size limit")

ofscraper/download/downloadbatch.py

@@ -29,6 +29,7 @@
 import ofscraper.utils.logs.other as other_logs
 import ofscraper.utils.logs.stdout as stdout_logs
 import ofscraper.utils.manager as manager_
+import ofscraper.utils.settings as settings
 import ofscraper.utils.system.system as system
 from ofscraper.download.alt_downloadbatch import alt_download
 from ofscraper.download.common import (
@@ -225,9 +226,7 @@ def process_dicts(username, model_id, filtered_medialist):
 
 def queue_process(pipe_, overall_progress, job_progress, task1, total):
     count = 0
-    downloadprogress = (
-        read_args.retriveArgs().downloadbars or config_data.get_show_downloadprogress()
-    )
+    downloadprogress = settings.get_download_bar()
     # shared globals
 
     while True:
@@ -301,7 +300,7 @@ def queue_process(pipe_, overall_progress, job_progress, task1, total):
 
 
 def get_mediasplits(medialist):
-    user_count = read_args.retriveArgs().downloadthreads or config_data.get_threads()
+    user_count = settings.get_threads()
     final_count = max(min(user_count, system.getcpu_count(), len(medialist) // 5), 1)
     return more_itertools.divide(final_count, medialist)
 

ofscraper/download/keyhelpers.py

@@ -26,6 +26,7 @@
 import ofscraper.utils.cache as cache
 import ofscraper.utils.config.data as config_data
 import ofscraper.utils.constants as constants
+import ofscraper.utils.settings as settings
 from ofscraper.download.common import get_medialog
 
 log = None
@@ -39,7 +40,7 @@ def setLog(input_):
 async def un_encrypt(item, c, ele, input_=None):
     setLog(input_ or common.log)
     key = None
-    keymode = read_args.retriveArgs().key_mode or config_data.get_key_mode() or "cdrm"
+    keymode = settings.get_key_mode()
     past_key = await asyncio.get_event_loop().run_in_executor(
         common.cache_thread, partial(cache.get, ele.license)
     )

ofscraper/download/main_download.py

@@ -130,7 +130,7 @@ async def main_download_downloader(c, ele, username, model_id, progress):
                 common.attempt.set(common.attempt.get(0) + 1)
                 try:
                     placeholderObj = placeholder.Placeholders()
-                    placeholderObj.getDirs(ele, username, model_id)
+                    await placeholderObj.getDirs(ele, username, model_id)
                     placeholderObj.tempfilename = f"{ele.filename}_{ele.id}.part"
 
                     data = await asyncio.get_event_loop().run_in_executor(

ofscraper/filters/media/helpers.py

@@ -6,6 +6,7 @@
 
 import ofscraper.utils.args.read as read_args
 import ofscraper.utils.config.data as config_data
+import ofscraper.utils.settings as settings
 
 log = logging.getLogger("shared")
 
@@ -55,14 +56,12 @@ def timeline_array_filter(posts):
 
 
 def post_count_filter(media):
-    count = (
-        read_args.retriveArgs().max_count or config_data.get_max_post_count() or None
-    )
+    count = settings.get_max_post_count() or None
     return media[:count]
 
 
 def posts_type_filter(media):
-    filtersettings = read_args.retriveArgs().mediatype or config_data.get_filter()
+    filtersettings = settings.get_mediatypes()
     if isinstance(filtersettings, str):
         filtersettings = filtersettings.split(",")
     if isinstance(filtersettings, list):

ofscraper/models/retriver.py

@@ -10,7 +10,7 @@
 import ofscraper.utils.me as me_util
 
 
-def get_models(main=False) -> list:
+def get_models() -> list:
     """
     Get user's subscriptions in form of a list.
     """

ofscraper/models/selector.py

@@ -13,6 +13,7 @@
 import ofscraper.utils.args.write as write_args
 import ofscraper.utils.constants as constants
 import ofscraper.utils.manager as manager
+import ofscraper.utils.settings as settings
 
 ALL_SUBS = None
 PARSED_SUBS = None
@@ -31,9 +32,7 @@ def set_ALL_SUBS_DICT(subsDict=None):
     if subsDict and isinstance(subsDict, dict):
         ALL_SUBS_DICT = subsDict
     else:
-        subList = subsDict or ALL_SUBS
-        if not subList:
-            all_subs_helper()
+        subList = subsDict or ALL_SUBS or []
         ALL_SUBS_DICT = {}
         [ALL_SUBS_DICT.update({ele.name: ele}) for ele in subList]
 
@@ -84,14 +83,14 @@ def getselected_usernames(rescan=False, reset=False):
     return PARSED_SUBS
 
 
-def all_subs_helper(refetch=True, main=False):
+def all_subs_helper(refetch=True, main=False, check=True):
     global ALL_SUBS
     if bool(ALL_SUBS) and not refetch:
         return
     while True:
-        ALL_SUBS = retriver.get_models(main)
-        if len(ALL_SUBS) > 0:
-            set_ALL_SUBS_DICTVManger()
+        ALL_SUBS = retriver.get_models()
+        if len(ALL_SUBS) > 0 or not check:
+            set_ALL_SUBS_DICTVManger(subsDict=ALL_SUBS)
             break
         elif len(ALL_SUBS) == 0:
             print("No accounts found during scan")
@@ -109,12 +108,12 @@ def parsed_subscriptions_helper(reset=False):
         args.username = None
         write_args.setArgs(args)
     if not bool(args.username):
-        selectedusers = retriver.get_selected_model(filterNSort((ALL_SUBS)))
+        selectedusers = retriver.get_selected_model(filterNSort())
         read_args.retriveArgs().username = list(map(lambda x: x.name, selectedusers))
         PARSED_SUBS = selectedusers
         write_args.setArgs(args)
     elif "ALL" in args.username:
-        PARSED_SUBS = filterNSort(ALL_SUBS)
+        PARSED_SUBS = filterNSort()
     elif args.username:
         usernameset = set(args.username)
         PARSED_SUBS = list(filter(lambda x: x.name in usernameset, ALL_SUBS))
@@ -123,6 +122,7 @@ def parsed_subscriptions_helper(reset=False):
 
 def setfilter(forced=False):
     global args
+    global ALL_SUBS
     while True:
         choice = prompts.decide_filters_menu()
         if choice == "modelList":
@@ -137,11 +137,25 @@ def setfilter(forced=False):
             args = prompts.modify_active_prompt(read_args.retriveArgs())
         elif choice == "price":
             args = prompts.modify_prices_prompt(read_args.retriveArgs())
+        elif choice == "list":
+            old_args = read_args.retriveArgs()
+            old_blacklist = old_args.black_list
+            old_list = old_args.user_list
+            args = prompts.modify_list_prompt(old_args)
+            if not list(sorted(old_blacklist)) == list(
+                sorted(args.black_list)
+            ) or not list(sorted(old_list)) == list(sorted(args.user_list)):
+                print("Updating Models")
+                all_subs_helper(check=False)
+        write_args.setArgs(args)
 
 
-def filterNSort(usernames):
+def filterNSort():
+    global ALL_SUBS
     while True:
         # paid/free
+        usernames = ALL_SUBS
+
         log.debug(f"username count no filters: {len(usernames)}")
         filterusername = subtype.subType(usernames)
         filterusername = price.pricePaidFreeFilterHelper(filterusername)
@@ -158,8 +172,12 @@ def filterNSort(usernames):
             f"""You have filtered the user list to zero
 Change the filter settings to continue
 
+Active userlist : {settings.get_userlist()}
+Active blacklist : {settings.get_blacklist()}
+
 Sub Status: {read_args.retriveArgs().sub_status or 'No Filter'}
 Renewal Status: {read_args.retriveArgs().renewal or 'No Filter'}
+
 Promo Price Filter: {read_args.retriveArgs().promo_price or 'No Filter'}
 Current Price Filter: {read_args.retriveArgs().current_price or 'No Filter'}
 Current Price Filter: {read_args.retriveArgs().current_price or 'No Filter'}

ofscraper/prompts/helpers/prompt_helpers.py

@@ -7,7 +7,9 @@
 
 import ofscraper.models.selector as userselector
 import ofscraper.prompts.helpers.model_helpers as modelHelpers
+import ofscraper.prompts.prompt_strings as prompt_strings
 import ofscraper.utils.args.read as read_args
+import ofscraper.utils.config.data as config_data
 
 console = Console()
 
@@ -108,7 +110,7 @@ def model_funct(prompt):
     oldargs = copy.deepcopy(vars(read_args.retriveArgs()))
     userselector.setfilter()
     if oldargs != vars(read_args.retriveArgs()):
-        models = userselector.filterNSort(userselector.ALL_SUBS)
+        models = userselector.filterNSort()
     choices = list(
         map(
             lambda x: modelHelpers.model_selectorHelper(x[0], x[1]),
@@ -131,3 +133,23 @@ def model_funct(prompt):
     )
     prompt.content_control._format_choices()
     return prompt
+
+
+def user_list(model_str):
+    model_list = re.split("(,|\n)", model_str)
+    model_list = map(lambda x: re.sub("[^a-zA-Z0-9 ]", "", x), model_list)
+    model_list = filter(lambda x: len(x) != 0, model_list)
+    return model_list
+
+
+def get_list_details(white=True):
+    return (
+        f"""
+    {prompt_strings.LIST_PROMPT_INFO}
+Default User Lists : {config_data.get_default_userlist() or 'No Default List Selected'}
+    """
+        if white
+        else f"""{prompt_strings.LIST_PROMPT_INFO}
+Default Black Lists : {config_data.get_default_blacklist() or 'No Default List Selected'}
+    """
+    )

ofscraper/prompts/promptConvert.py

@@ -21,12 +21,14 @@ def inner(*args, **kwargs):
                 filter(
                     lambda x: len(x) > 0,
                     [
-                        inspect.cleandoc(f"{kwargs.pop('option_instruction', '')}"),
                         inspect.cleandoc(
-                            f"{kwargs.get('long_instruction', prompt_strings.KEY_BOARD)}"
+                            f"{kwargs.pop('option_instruction', '')}".upper()
                         ),
                         inspect.cleandoc(
-                            f"{kwargs.pop('more_instruction', '') or kwargs.pop('more_instructions', '')}"
+                            f"{kwargs.get('long_instruction', prompt_strings.KEY_BOARD)}".upper()
+                        ),
+                        inspect.cleandoc(
+                            f"{kwargs.pop('more_instruction', '') or kwargs.pop('more_instructions', '')}".upper()
                         ),
                     ],
                 )

ofscraper/prompts/prompt_groups/model.py

@@ -56,7 +56,7 @@ def decide_filters_menu() -> int:
     name = "modelList"
     modelChoice = [*constants.getattr("modelPrompt")]
     modelChoice.insert(4, Separator())
-    modelChoice.insert(6, Separator())
+    modelChoice.insert(7, Separator())
     questions = promptClasses.batchConverter(
         *[
             {
@@ -332,6 +332,34 @@ def modify_sort_prompt(args):
     return args
 
 
+def modify_list_prompt(args):
+    answer = promptClasses.batchConverter(
+        *[
+            {
+                "type": "input",
+                "name": "user_list",
+                "message": "Change User List",
+                "default": ",".join(args.user_list or []),
+                "multiline": True,
+                "filter": lambda x: prompt_helpers.user_list(x),
+                "option_instruction": prompt_helpers.get_list_details(True),
+            },
+            {
+                "type": "input",
+                "name": "black_list",
+                "message": "Change Black List",
+                "default": ",".join(args.black_list or []),
+                "multiline": True,
+                "filter": lambda x: prompt_helpers.user_list(x),
+                "option_instruction": prompt_helpers.get_list_details(False),
+            },
+        ],
+    )
+    args.user_list = list(answer["user_list"])
+    args.black_list = list(answer["black_list"])
+    return args
+
+
 def reset_username_prompt() -> bool:
     name = "reset username"
     answer = promptClasses.batchConverter(