Merge pull request #22 from saveweb/Ovler-Young/fix-403-error

Fix 403 error handling in upload
saveweb · Nov 22, 2024 · 65873f4 · 65873f4
2 parents e004fe0 + 03b4b8e
commit 65873f4
Showing 1 changed file with 43 additions and 8 deletions.
diff --git a/biliarchiver/_biliarchiver_upload_bvid.py b/biliarchiver/_biliarchiver_upload_bvid.py
@@ -5,7 +5,7 @@
 from typing import List
 from urllib.parse import urlparse
 from internetarchive import get_item
-from requests import Response
+from requests import Response, HTTPError
 from rich import print
 from pathlib import Path
 from shutil import rmtree
@@ -287,13 +287,48 @@ def _upload_bvid(
                 print("Removed XML illegal characters from metadata, cleaned metadata:")
                 print(new_md)
 
-            r = item.modify_metadata(
-                metadata=new_md,
-                access_key=access_key,
-                secret_key=secret_key,
-            )
-            assert isinstance(r, Response)
-            r.raise_for_status()
+            try:
+                r = item.modify_metadata(
+                    metadata=new_md,
+                    access_key=access_key,
+                    secret_key=secret_key,
+                )
+                assert isinstance(r, Response)
+                r.raise_for_status()
+            except HTTPError as e:
+                if e.response.status_code == 403:
+                    print(f"403 Forbidden error encountered for {remote_identifier}. Retrying with description as title.")
+                    new_md["description"] = md["title"]
+                    try:
+                        r = item.modify_metadata(
+                            metadata=new_md,
+                            access_key=access_key,
+                            secret_key=secret_key,
+                        )
+                        assert isinstance(r, Response)
+                        r.raise_for_status()
+                    except HTTPError as e:
+                        if e.response.status_code == 403:
+                            print(f"403 Forbidden error encountered again for {remote_identifier}. Retrying with empty description.")
+                            new_md["description"] = ""
+                            try:
+                                r = item.modify_metadata(
+                                    metadata=new_md,
+                                    access_key=access_key,
+                                    secret_key=secret_key,
+                                )
+                                assert isinstance(r, Response)
+                                r.raise_for_status()
+                            except HTTPError as e:
+                                if e.response.status_code == 403:
+                                    print(f"403 Forbidden error encountered again for {remote_identifier}. No more retries will be attempted.")
+                                else:
+                                    raise e
+                        else:
+                            raise e
+                else:
+                    raise e
+
         with open(
             f"{videos_basepath}/{local_identifier}/_uploaded.mark",
             "w",