Skip to content

Commit

Permalink
placeholder init change and fix metadata
Browse files Browse the repository at this point in the history
  • Loading branch information
datawhores committed Mar 31, 2024
1 parent 48c2cbf commit 8ea580e
Show file tree
Hide file tree
Showing 5 changed files with 27 additions and 30 deletions.
13 changes: 6 additions & 7 deletions ofscraper/download/alt_download.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,7 @@ async def alt_download(c, ele, username, model_id, progress):
common_globals.log.debug(
f"{get_medialog(ele)} Downloading with protected media downloader"
)
sharedPlaceholderObj = placeholder.Placeholders(ele, "mp4")
await sharedPlaceholderObj.init()
sharedPlaceholderObj = await placeholder.Placeholders(ele, "mp4").init()
common_globals.log.debug(f"{get_medialog(ele)} download url: {get_url_log(ele)}")
if read_args.retriveArgs().metadata != None:
return await metadata(
Expand All @@ -89,10 +88,9 @@ async def alt_download(c, ele, username, model_id, progress):


async def handle_result(sharedPlaceholderObj, ele, audio, video, username, model_id):
tempPlaceholder = placeholder.tempFilePlaceholder(
tempPlaceholder = await placeholder.tempFilePlaceholder(
ele, f"temp_{ele.id or await ele.final_filename}.mp4"
)
await tempPlaceholder.init()
).init()
temp_path = tempPlaceholder.tempfilepath
temp_path.unlink(missing_ok=True)
t = subprocess.run(
Expand Down Expand Up @@ -172,8 +170,9 @@ async def media_item_keys(c, audio, video, ele):

async def alt_download_downloader(item, c, ele, progress):
downloadspace(mediatype=ele.mediatype)
placeholderObj = placeholder.tempFilePlaceholder(ele, f"{item['name']}.part")
await placeholderObj.init()
placeholderObj = await placeholder.tempFilePlaceholder(
ele, f"{item['name']}.part"
).init()
item["path"] = placeholderObj.tempfilepath
item["total"] = None
async for _ in AsyncRetrying(
Expand Down
13 changes: 6 additions & 7 deletions ofscraper/download/alt_downloadbatch.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,7 @@ async def alt_download(c, ele, username, model_id):
common_globals.innerlog.get().debug(
f"{get_medialog(ele)} download url: {get_url_log(ele)}"
)
sharedPlaceholderObj = placeholder.Placeholders(ele, "mp4")
await sharedPlaceholderObj.init()
sharedPlaceholderObj = await placeholder.Placeholders(ele, "mp4").init()
if read_args.retriveArgs().metadata != None:
return await metadata(
c, ele, username, model_id, placeholderObj=sharedPlaceholderObj
Expand All @@ -76,10 +75,9 @@ async def alt_download(c, ele, username, model_id):


async def handle_result(sharedPlaceholderObj, ele, audio, video, username, model_id):
tempPlaceholder = placeholder.tempFilePlaceholder(
tempPlaceholder = await placeholder.tempFilePlaceholder(
ele, f"temp_{ele.id or await ele.final_filename}.mp4"
)
await tempPlaceholder.init()
).init()
temp_path = tempPlaceholder.tempfilepath

temp_path.unlink(missing_ok=True)
Expand Down Expand Up @@ -168,8 +166,9 @@ async def alt_download_downloader(
ele,
):
downloadspace(mediatype=ele.mediatype)
placeholderObj = placeholder.tempFilePlaceholder(ele, f"{item['name']}.part")
await placeholderObj.init()
placeholderObj = await placeholder.tempFilePlaceholder(
ele, f"{item['name']}.part"
).init()
item["path"] = placeholderObj.tempfilepath
item["total"] = None

Expand Down
10 changes: 6 additions & 4 deletions ofscraper/download/common/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,7 @@ async def metadata(c, ele, username, model_id, placeholderObj=None):
)
elif download_data and download_data.get("content-type"):
content_type = download_data.get("content-type").split("/")[-1]
placeholderObj = placeholder.Placeholders(ele)
await placeholderObj.set_trunicated_filepath(ele, content_type)
placeholderObj = await placeholder.Placeholders(ele, content_type).init()
if ele.id:
await operations.update_media_table(
ele,
Expand Down Expand Up @@ -134,7 +133,10 @@ async def metadata_helper(c, ele, placeholderObj=None):
content_type = "mp4"
elif not content_type and ele.mediatype.lower() == "images":
content_type = "jpg"
placeholderObj = placeholderObj or placeholder.Placeholders(ele)
placeholderObj = (
placeholderObj
or await placeholder.Placeholders(ele, content_type).init()
)
return placeholderObj

else:
Expand All @@ -148,5 +150,5 @@ async def meta_data_placeholder(ele, username, model_id):
content_type = "jpg"
elif ele.mediatype.lower() == "audios":
content_type = "mp3"
placeholderObj = placeholder.Placeholders()
placeholderObj = await placeholder.Placeholders(ele, content_type).init()
return placeholderObj
8 changes: 3 additions & 5 deletions ofscraper/download/main_download.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,10 +126,9 @@ async def handle_result(result, ele, username, model_id):

async def main_download_downloader(c, ele, progress):
downloadspace(mediatype=ele.mediatype)
tempholderObj = placeholder.tempFilePlaceholder(
tempholderObj = await placeholder.tempFilePlaceholder(
ele, f"{await ele.final_filename}_{ele.id}.part"
)
await tempholderObj.init()
).init()
async for _ in AsyncRetrying(
stop=stop_after_attempt(constants.getattr("DOWNLOAD_RETRIES")),
wait=wait_random(
Expand Down Expand Up @@ -185,9 +184,8 @@ async def alt_data_handler(c, tempholderObj, ele, progress):
async def main_data_handler(data, c, tempholderObj, ele, progress):
content_type = data.get("content-type").split("/")[-1]
total = int(data.get("content-length"))
placeholderObj = placeholder.Placeholders(ele, content_type)
placeholderObj = await placeholder.Placeholders(ele, content_type).init()
resume_size = get_resume_size(tempholderObj, mediatype=ele.mediatype)
await placeholderObj.init()
# other
if await check_forced_skip(ele, total) == 0:
path_to_file_logger(placeholderObj, ele)
Expand Down
13 changes: 6 additions & 7 deletions ofscraper/download/main_downloadbatch.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,10 +123,9 @@ async def handle_result(result, ele, username, model_id):

async def main_download_downloader(c, ele):
downloadspace(mediatype=ele.mediatype)
tempholderObj = placeholder.tempFilePlaceholder(
tempholderObj = await placeholder.tempFilePlaceholder(
ele, f"{await ele.final_filename}_{ele.id}.part"
)
await tempholderObj.init()
).init()
async for _ in AsyncRetrying(
stop=stop_after_attempt(constants.getattr("DOWNLOAD_RETRIES")),
wait=wait_random(
Expand Down Expand Up @@ -184,9 +183,8 @@ async def alt_data_handler(c, ele, tempholderObj):
async def main_data_handler(data, c, ele, tempholderObj):
content_type = data.get("content-type").split("/")[-1]
total = int(data.get("content-length"))
placeholderObj = placeholder.Placeholders(ele, content_type)
placeholderObj = await placeholder.Placeholders(ele, content_type).init()
resume_size = get_resume_size(tempholderObj, mediatype=ele.mediatype)
await placeholderObj.init()
# other
if await check_forced_skip(ele, total) == 0:
path_to_file_logger(placeholderObj, ele, common_globals.innerlog.get())
Expand Down Expand Up @@ -251,8 +249,9 @@ async def send_req_inner(c, ele, tempholderObj, placeholderObj=None, total=None)
content_type = r.headers.get("content-type").split("/")[-1]
content_type = get_unknown_content_type(ele)
if not placeholderObj:
placeholderObj = placeholder.Placeholders(ele, content_type)
await placeholderObj.init()
placeholderObj = await placeholder.Placeholders(
ele, content_type
).init()
path_to_file_logger(placeholderObj, ele, common_globals.innerlog.get())
if await check_forced_skip(ele, total) == 0:
total = 0
Expand Down

0 comments on commit 8ea580e

Please sign in to comment.