Skip to content

Commit

Permalink
fix manual and check mode
Browse files Browse the repository at this point in the history
  • Loading branch information
datawhores committed Jan 25, 2024
1 parent 323620b commit 8e2cdad
Show file tree
Hide file tree
Showing 7 changed files with 22 additions and 9 deletions.
8 changes: 5 additions & 3 deletions ofscraper/api/archive.py
Original file line number Diff line number Diff line change
Expand Up @@ -215,9 +215,11 @@ async def get_archived_media(model_id, username, forced_after=None, rescan=None)
or cache.get("{model_id}_scrape_archived")
and not read_args.retriveArgs().after
)
after = after = (
0 if rescan else forced_after or get_after(model_id, username)
)
if forced_after != None:
after = forced_after
else:
after = get_after() if rescan == None else 0

log.info(
f"""
Setting initial archived scan date for {username} to {arrow.get(after).format('YYYY.MM.DD')}
Expand Down
8 changes: 5 additions & 3 deletions ofscraper/api/messages.py
Original file line number Diff line number Diff line change
Expand Up @@ -106,9 +106,11 @@ async def get_messages(model_id, username, forced_after=None, rescan=None):
or cache.get("{model_id}_scrape_messages")
and not read_args.retriveArgs().after
)
after = after = (
0 if rescan else forced_after or get_after(model_id, username)
)

if forced_after != None:
after = forced_after
else:
after = get_after() if rescan == None else 0
log.debug(f"Messages after = {after}")

log.debug(f"Messages before = {before}")
Expand Down
5 changes: 4 additions & 1 deletion ofscraper/api/timeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,10 @@ async def get_timeline_media(model_id, username, forced_after=None, rescan=None)
cache.get("{model_id}_scrape_timeline")
and not read_args.retriveArgs().after
)
after = after = 0 if rescan else forced_after or get_after(model_id, username)
if forced_after != None:
after = forced_after
else:
after = get_after() if rescan == None else 0

log.info(
f"""
Expand Down
6 changes: 5 additions & 1 deletion ofscraper/commands/check.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import ofscraper.commands.manual as manual
import ofscraper.db.operations as operations
import ofscraper.download.downloadnormal as downloadnormal
import ofscraper.models.selector as selector
import ofscraper.utils.args.read as read_args
import ofscraper.utils.args.write as write_args
import ofscraper.utils.auth as auth
Expand Down Expand Up @@ -58,10 +59,13 @@ def process_download_cart():
if process_download_cart.counter == 0:
if not network.check_cdm():
log.info(
"error was raised by cdm checker\nncdm will not be check again\n\n"
"error was raised by cdm checker\ncdm will not be check again\n\n"
)
else:
log.info("cdm checker was fine\ncdm will not be check again\n\n")
# should be done once before downloads
log.info("Getting Models")
selector.all_subs_helper()

process_download_cart.counter = process_download_cart.counter + 1
log.info("Getting items from queue")
Expand Down
1 change: 1 addition & 0 deletions ofscraper/commands/manual.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ def manual_download(urls=None):
args = read_args.retriveArgs()
args.dupe = True
write_args.setArgs(args)
# should be done before downloads
selector.all_subs_helper()
for value in media_dict.values():
if len(value) == 0:
Expand Down
2 changes: 1 addition & 1 deletion ofscraper/download/main_download.py
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@ async def main_download_datahandler(r, progress, ele, placeholderObj, total):
data.get_show_downloadprogress() or read_args.retriveArgs().downloadbars
)
task1 = progress.add_task(
f"{(pathstr[:constants.getattr('PATH_STR_MAX)')] + '....') if len(pathstr) > constants.getattr('PATH_STR_MAX') else pathstr}\n",
f"{(pathstr[:constants.getattr('PATH_STR_MAX')] + '....') if len(pathstr) > constants.getattr('PATH_STR_MAX') else pathstr}\n",
total=total,
visible=True if downloadprogress else False,
)
Expand Down
1 change: 1 addition & 0 deletions ofscraper/models/selector.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@


def get_model_fromParsed(name):
global ALL_SUBS
if not ALL_SUBS:
all_subs_helper()
modelObjs = list(filter(lambda x: x.name == name, ALL_SUBS))
Expand Down

0 comments on commit 8e2cdad

Please sign in to comment.