Skip to content

Commit

Permalink
add maxcount
Browse files Browse the repository at this point in the history
  • Loading branch information
datawhores committed Feb 13, 2024
1 parent 50a9cf1 commit 6c1a0ae
Show file tree
Hide file tree
Showing 5 changed files with 34 additions and 2 deletions.
1 change: 1 addition & 0 deletions ofscraper/const/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
DYNAMIC_DEFAULT = "deviint"
TEXT_TYPE_DEFAULT = "letter"
TRUNCATION_DEFAULT = True
MAX_COUNT_DEFAULT = 0
TEMP_FOLDER_DEFAULT = None
APPEND_DEFAULT = True
RESPONSE_TYPE_DEFAULT = {
Expand Down
15 changes: 13 additions & 2 deletions ofscraper/download/download.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ def medialist_filter(medialist, model_id, username):

def download_picker(username, model_id, medialist):
medialist = medialist_filter(medialist, model_id, username)
maxCount = get_max_count()
medialist = medialist if maxCount == 0 else medialist[:maxCount]
if len(medialist) == 0:
logging.getLogger("shared").error(
f"[bold]{username}[/bold] ({0} photos, {0} videos, {0} audios, {0} skipped, {0} failed)"
Expand All @@ -44,9 +46,18 @@ def download_picker(username, model_id, medialist):
>= config_data.get_download_semaphores()
* constants.getattr("DOWNLOAD_THREAD_MIN")
)
and read_args.retriveArgs().downloadthreads != 0
and config_data.get_threads() != 0
and not_solo_thread()
):
return batchdownloader.process_dicts(username, model_id, medialist)
else:
return normaldownloader.process_dicts(username, model_id, medialist)


def get_max_count():
return read_args.retriveArgs().max_count or config_data.get_max_post_count()


def not_solo_thread():
return (
read_args.retriveArgs().downloadthreads != 0 and config_data.get_threads() != 0
)
8 changes: 8 additions & 0 deletions ofscraper/utils/args/parse.py
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,14 @@ def create_parser(input=None):
required=False,
action="store_true",
)
post.add_argument(
"-xc",
"--max-count",
help="Max number of post to download for a specific model",
default=0,
required=False,
type=str,
)
group10 = post.add_mutually_exclusive_group()
group10.add_argument(
"-to",
Expand Down
11 changes: 11 additions & 0 deletions ofscraper/utils/config/data.py
Original file line number Diff line number Diff line change
Expand Up @@ -587,3 +587,14 @@ def get_truncation(config=None):
else config.get("truncation_default")
)
return val if val is not None else constants_attr.getattr("TRUNCATION_DEFAULT")


@wrapper.config_reader
def get_max_post_count(config=None):
if config == False:
return constants.MAX_COUNT_DEFAULT
if config.get("max_post_count") != None:
return config.get("max_post_count")
elif config.get("download_options", {}).get("max_post_count") != None:
return config.get("download_options", {}).get("max_post_count")
return constants_attr.getattr("MAX_COUNT_DEFAULT")
1 change: 1 addition & 0 deletions ofscraper/utils/config/schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ def get_current_config_schema(config: dict = None) -> dict:
"filter": data.get_filter(config=config),
"auto_resume": data.get_part_file_clean(config=config),
"system_free_min": data.get_system_freesize(config=config),
"max_post_count": data.get_max_post_count(config=config),
},
"binary_options": {
"mp4decrypt": data.get_mp4decrypt(config=config),
Expand Down

0 comments on commit 6c1a0ae

Please sign in to comment.