diff --git a/ofscraper/actions/process.py b/ofscraper/actions/process.py index 0652bb022..75baac370 100644 --- a/ofscraper/actions/process.py +++ b/ofscraper/actions/process.py @@ -170,6 +170,7 @@ def normal_post_process(): model_id = ele.id operations.table_init_create(model_id, ele.name) combined_urls, posts = asyncio.run(OF.process_areas(ele, model_id)) + return download.download_process( ele.name, model_id, combined_urls, posts=posts ) diff --git a/ofscraper/api/highlights.py b/ofscraper/api/highlights.py index eb476b2e5..f5191eeef 100644 --- a/ofscraper/api/highlights.py +++ b/ofscraper/api/highlights.py @@ -220,6 +220,11 @@ async def scrape_stories(c, user_id, job_progress=None) -> list: @run async def get_highlight_post_progress(model_id, c=None): + highlightLists = await get_highlight_list_progress(model_id, c) + return await get_highlights_via_list_progress(highlightLists, c) + + +async def get_highlight_list_progress(model_id, c=None): global sem sem = semaphoreDelayed(1) @@ -265,8 +270,14 @@ async def get_highlight_post_progress(model_id, c=None): log.traceback_(traceback.format_exc()) tasks = new_tasks - tasks = [] + overall_progress.remove_task(page_task) + return highlightLists + +async def get_highlights_via_list_progress(highlightLists, c=None): + job_progress = progress_utils.highlights_progress + overall_progress = progress_utils.overall_progress + tasks = [] [ tasks.append( asyncio.create_task(scrape_highlights(c, i, job_progress=job_progress)) @@ -303,9 +314,6 @@ async def get_highlight_post_progress(model_id, c=None): log.traceback_(E) log.traceback_(traceback.format_exc()) tasks = new_tasks - overall_progress.remove_task(page_task) - progress_utils.highlights_layout.visible = False - log.trace( "highlight raw unduped {posts}".format( posts="\n\n".join( @@ -324,11 +332,18 @@ async def get_highlight_post_progress(model_id, c=None): log.debug( f"[bold]highlight Count with Dupes[/bold] {len(list(outdict.values()))} found" ) + overall_progress.remove_task(page_task) + progress_utils.highlights_layout.visible = False return list(outdict.values()) @run async def get_highlight_post(model_id, c=None): + highlightList = await get_highlight_list(model_id, c) + return await get_highlights_via_list(highlightList) + + +async def get_highlight_list(model_id, c=None): global sem sem = semaphoreDelayed(1) @@ -363,6 +378,10 @@ async def get_highlight_post(model_id, c=None): log.traceback_(traceback.format_exc()) tasks = new_tasks + return highlightLists + + +async def get_highlights_via_list(highlightLists, c): tasks = [] [ tasks.append(asyncio.create_task(scrape_highlights(c, i, job_progress=None))) diff --git a/ofscraper/const/req.py b/ofscraper/const/req.py index a94c27df0..2279b16e6 100644 --- a/ofscraper/const/req.py +++ b/ofscraper/const/req.py @@ -25,5 +25,5 @@ LOGIN_NUM_TRIES = 3 MAX_REQUEST_WORKERS = 20 API_REQUEST_THREADONLY = ["Windows", "Linux", "Mac"] -API_MAX_AREAS = 3 +API_MAX_AREAS = 2 API_TIMEOUT_PER_TASKS = 35