Skip to content

Commit

Permalink
fix some double uploads
Browse files Browse the repository at this point in the history
  • Loading branch information
datawhores committed Mar 29, 2024
1 parent e7e1aec commit 3619a1f
Showing 1 changed file with 1 addition and 13 deletions.
14 changes: 1 addition & 13 deletions ofscraper/actions/scraper.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ async def process_messages(model_id, username, c):
[output.extend(message.media) for message in messages_]
log.debug(f"[bold]Messages media count[/bold] {len(output)}")

await operations.batch_mediainsert(
await operations.write_messages_table(
output,
model_id=model_id,
username=username,
Expand Down Expand Up @@ -242,12 +242,6 @@ async def process_timeline_posts(model_id, username, c):
[output.extend(post.media) for post in timeline_posts]
log.debug(f"[bold]Timeline media count without locked[/bold] {len(output)}")

await operations.batch_mediainsert(
output,
model_id=model_id,
username=username,
downloaded=False,
)
await operations.batch_mediainsert(
output,
model_id=model_id,
Expand Down Expand Up @@ -342,12 +336,6 @@ async def process_pinned_posts(model_id, username, c):
[output.extend(post.media) for post in pinned_posts]
log.debug(f"[bold]Pinned media count without locked[/bold] {len(output)}")

await operations.batch_mediainsert(
output,
model_id=model_id,
username=username,
downloaded=False,
)
await operations.batch_mediainsert(
output,
model_id=model_id,
Expand Down

0 comments on commit 3619a1f

Please sign in to comment.