Skip to content

Commit

Permalink
make some changes to paid failback use
Browse files Browse the repository at this point in the history
  • Loading branch information
datawhores committed Mar 11, 2024
1 parent 3dde4ac commit bd16f80
Showing 1 changed file with 30 additions and 10 deletions.
40 changes: 30 additions & 10 deletions ofscraper/commands/manual.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import ofscraper.api.paid as paid
import ofscraper.api.profile as profile
import ofscraper.api.timeline as timeline
import ofscraper.classes.media as media_
import ofscraper.classes.posts as posts_
import ofscraper.classes.sessionbuilder as sessionbuilder
import ofscraper.db.operations as operations
Expand Down Expand Up @@ -68,21 +69,21 @@ def get_media_from_urls(urls):
value = timeline.get_individual_post(
postid, c=sessionbuilder.sessionBuilder(backend="httpx")
)
media_dict.update(get_all_media(model_id, value))
media_dict.update(get_all_media(postid, model_id, value))
post_dict.update(get_post_item(model_id, value))
elif type == "msg":
model_id = model
value = messages_.get_individual_post(
model_id, postid, c=sessionbuilder.sessionBuilder(backend="httpx")
)
media_dict.update(get_all_media(model_id, value))
media_dict.update(get_all_media(postid, model_id, value))
post_dict.update(get_post_item(model_id, value))
elif type == "msg2":
model_id = user_name_dict.get(model) or profile.get_id(model)
value = messages_.get_individual_post(
model_id, postid, c=sessionbuilder.sessionBuilder(backend="httpx")
)
media_dict.update(get_all_media(model_id, value))
media_dict.update(get_all_media(postid, model_id, value))
post_dict.update(get_post_item(model_id, value))
elif type == "unknown":
value = (
Expand All @@ -92,7 +93,7 @@ def get_media_from_urls(urls):
or {}
)
model_id = value.get("author", {}).get("id")
media_dict.update(get_all_media(model_id, value))
media_dict.update(get_all_media(postid, model_id, value))
post_dict.update(get_post_item(model_id, value))
elif type == "highlights":
value = (
Expand All @@ -102,7 +103,7 @@ def get_media_from_urls(urls):
or {}
)
model_id = value.get("userId")
media_dict.update(get_all_media(model_id, value, "highlights"))
media_dict.update(get_all_media(postid, model_id, value, "highlights"))
post_dict.update(get_post_item(model_id, value, "highlights"))
# special case
elif type == "stories":
Expand All @@ -113,7 +114,7 @@ def get_media_from_urls(urls):
or {}
)
model_id = value.get("userId")
media_dict.update(get_all_media(model_id, value, "stories"))
media_dict.update(get_all_media(postid, model_id, value, "stories"))
post_dict.update(get_post_item(model_id, value, "stories"))
# special case
return media_dict, post_dict
Expand All @@ -131,7 +132,7 @@ def get_post_item(model_id, value, inputtype=None):
return {post.id: post}


def get_all_media(model_id, value, inputtype=None):
def get_all_media(posts_id, model_id, value, inputtype=None):
media_dict = {}
value = value or {}
media = []
Expand All @@ -140,18 +141,37 @@ def get_all_media(model_id, value, inputtype=None):
user_name = profile.scrape_profile(model_id)["username"]
post_item = posts_.Post(value, model_id, user_name, responsetype=inputtype)
media = post_item.media
media = list(
filter(
lambda x: isinstance(x, media_.Media)
and (str(x.id) == str(posts_id) or str(x.postid) == str(posts_id)),
media,
)
)
if len(media) == 0:
media.extend(paid_failback(model_id, user_name))
media.extend(paid_failback(posts_id, model_id, user_name))
media_dict[model_id] = media
return media_dict


def paid_failback(id, username):
def paid_failback(post_id, model_id, username):
logging.getLogger("shared").debug(
"Using failback search because query return 0 media"
)
post_id = str(post_id)
data = paid.get_paid_posts(id, username) or []
return list(map(lambda x: posts_.Post(x, id, username, responsetype="paid"), data))
posts = list(
map(lambda x: posts_.Post(x, model_id, username, responsetype="paid"), data)
)
output = []
[output.extend(post.media) for post in posts]
return list(
filter(
lambda x: isinstance(x, media_.Media)
and (str(x.id) == post_id or str(x.postid) == post_id),
output,
)
)


def get_info(url):
Expand Down

0 comments on commit bd16f80

Please sign in to comment.