fix(gnews): fixed prepare_gnews_url function

.github/workflows/pipeline.yml

@@ -14,7 +14,7 @@ jobs: # jobs. We will have two jobs (test and publish) with multiple steps.
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - uses: actions/checkout@v4
@@ -61,7 +61,7 @@ jobs: # jobs. We will have two jobs (test and publish) with multiple steps.
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.9"
       - name: Run image
         uses: abatilo/actions-poetry@v2
         with:

.github/workflows/pylint.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}

README.md

@@ -15,7 +15,7 @@ At the moment of the fork, in the original project were over 400 open issues, wh
 
 
 ## Python compatibility
-    - Python 3.8+ minimum
+    - Python 3.9+ minimum
 
 # Quick start
 

newspaper/google_news.py

@@ -6,14 +6,11 @@
 Install it using `pip install gnews` as a standalone package.
 """
 
-import base64
 from datetime import datetime
-import re
 from typing import Any, List, Optional
 from newspaper.article import Article
 from newspaper.source import Source
 
-
 try:
     import gnews
 except ImportError as e:
@@ -23,20 +20,14 @@
         "or pip install newspaper4k[gnews]\n"
         "or pip install newspaper4k[all]\n"
     ) from e
-
-# Some url encoding related constants
-_ENCODED_URL_PREFIX = "https://news.google.com/rss/articles/"
-_ENCODED_URL_PREFIX_WITH_CONSENT = (
-    "https://consent.google.com/m?continue=https://news.google.com/rss/articles/"
-)
-_ENCODED_URL_RE = re.compile(
-    rf"^{re.escape(_ENCODED_URL_PREFIX_WITH_CONSENT)}(?P<encoded_url>[^?]+)"
-)
-_ENCODED_URL_RE = re.compile(
-    rf"^{re.escape(_ENCODED_URL_PREFIX)}(?P<encoded_url>[^?]+)"
-)
-_DECODED_URL_RE = re.compile(rb'^\x08\x13".+?(?P<primary_url>http[^\xd2]+)\xd2\x01')
-
+try:
+    from googlenewsdecoder import new_decoderv1
+except ImportError as e:
+    raise ImportError(
+        "You must install googlenewsdecoder for fetching the actual url \n"
+        "Try pip install googlenewsdecoder\n"
+        "or pip install googlenewsdecoder\n"
+    ) from e
 
 class GoogleNewsSource(Source):
     """
@@ -220,21 +211,10 @@ def parse(self):
         """
 
         def prepare_gnews_url(url):
-            # There seems to be a case when we get a URL with consent.google.com
-            # see https://github.com/ranahaani/GNews/issues/62
-            # Also, the URL is directly decoded, no need to go through news.google.com
-
-            match = _ENCODED_URL_RE.match(url)
-            encoded_text = match.groupdict()["encoded_url"]
-            # Fix incorrect padding. Ref: https://stackoverflow.com/a/49459036/
-            encoded_text += "==="
-            decoded_text = base64.urlsafe_b64decode(encoded_text)
-
-            match = _DECODED_URL_RE.match(decoded_text)
-
-            primary_url = match.groupdict()["primary_url"]
-            primary_url = primary_url.decode()
-            return primary_url
+            decoded_url = new_decoderv1(url, interval=5)
+            if not decoded_url.get("status"):
+                raise ValueError("Failed to decode the URL")
+            return decoded_url.get("url")
 
         self.articles = []
         for res in self.gnews_results:

pyproject.toml

@@ -25,7 +25,7 @@ repository = "https://github.com/AndyTheFactory/newspaper4k"
 documentation = "https://newspaper4k.readthedocs.io/en/latest/"
 
 [tool.poetry.dependencies]
-python = "^3.8"
+python = "^3.9"
 beautifulsoup4 = ">=4.9.3"
 Pillow = ">=4.0.0"
 PyYAML = ">=5.1"
@@ -52,6 +52,7 @@ jieba = { version = ">=0.42.1", optional = true }
 indic-nlp-library = { version = ">=0.90", optional = true }
 cloudscraper = { version = ">=1.2.0", optional = true }
 gnews = { version = ">=0.3.6", optional = true }
+googlenewsdecoder = "^0.1.6"
 
 [tool.poetry.extras]
 zh = ["jieba"]

requirements.txt

@@ -1,29 +1,52 @@
-beautifulsoup4==4.12.2 ; python_version >= "3.8" and python_version < "4.0"
-certifi==2023.11.17 ; python_version >= "3.8" and python_version < "4.0"
-charset-normalizer==2.0.12 ; python_version >= "3.8" and python_version < "4.0"
-click==8.0.4 ; python_version >= "3.8" and python_version < "4.0"
-colorama==0.4.5 ; python_version >= "3.8" and python_version < "4.0" and platform_system == "Windows"
-feedparser==6.0.10 ; python_version >= "3.8" and python_version < "4.0"
-filelock==3.4.1 ; python_version >= "3.8" and python_version < "4.0"
-idna==3.4 ; python_version >= "3.8" and python_version < "4.0"
-jieba==0.42.1 ; python_version >= "3.8" and python_version < "4.0"
-joblib==1.1.1 ; python_version >= "3.8" and python_version < "4.0"
-lxml==4.9.3 ; python_version >= "3.8" and python_version < "4.0"
-nltk==3.6.7 ; python_version >= "3.8" and python_version < "4.0"
-pillow==8.4.0 ; python_version >= "3.8" and python_version < "4.0"
-pythainlp==2.3.2 ; python_version >= "3.8" and python_version < "4.0"
-python-crfsuite==0.9.9 ; python_version >= "3.8" and python_version < "4.0"
-python-dateutil==2.8.2 ; python_version >= "3.8" and python_version < "4.0"
-pyyaml==6.0.1 ; python_version >= "3.8" and python_version < "4.0"
-regex==2023.8.8 ; python_version >= "3.8" and python_version < "4.0"
-requests-file==1.5.1 ; python_version >= "3.8" and python_version < "4.0"
-requests==2.27.1 ; python_version >= "3.8" and python_version < "4.0"
-sgmllib3k==1.0.0 ; python_version >= "3.8" and python_version < "4.0"
-six==1.16.0 ; python_version >= "3.8" and python_version < "4.0"
-soupsieve==2.3.2.post1 ; python_version >= "3.8" and python_version < "4.0"
-tinydb==4.7.0 ; python_version >= "3.8" and python_version < "4.0"
-tinysegmenter==0.4 ; python_version >= "3.8" and python_version < "4.0"
-tldextract==3.1.2 ; python_version >= "3.8" and python_version < "4.0"
-tqdm==4.64.1 ; python_version >= "3.8" and python_version < "4.0"
-urllib3==1.26.18 ; python_version >= "3.8" and python_version < "4.0"
-typing-extensions==4.10.0 ; python_version >= "3.8" and python_version < "4.0"
+beautifulsoup4==4.9.3 ; python_version >= "3.9" and python_version < "4.0"
+certifi==2024.2.2 ; python_version >= "3.9" and python_version < "4.0"
+cfgv==3.4.0 ; python_version >= "3.9" and python_version < "4.0"
+charset-normalizer==2.0.12 ; python_version >= "3.9" and python_version < "4.0"
+click==8.1.7 ; python_version >= "3.9" and python_version < "4.0"
+codespell==2.2.6 ; python_version >= "3.9" and python_version < "4.0"
+colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0" and (sys_platform == "win32" or platform_system == "Windows")
+coverage==7.4.4 ; python_version >= "3.9" and python_version < "4.0"
+distlib==0.3.8 ; python_version >= "3.9" and python_version < "4.0"
+exceptiongroup==1.2.0 ; python_version >= "3.9" and python_version < "3.11"
+feedparser==6.0.11 ; python_version >= "3.9" and python_version < "4.0"
+filelock==3.13.1 ; python_version >= "3.9" and python_version < "4.0"
+googlenewsdecoder==0.1.6 ; python_version >= "3.9" and python_version < "4.0"
+identify==2.5.35 ; python_version >= "3.9" and python_version < "4.0"
+idna==3.6 ; python_version >= "3.9" and python_version < "4.0"
+iniconfig==2.0.0 ; python_version >= "3.9" and python_version < "4.0"
+joblib==1.3.2 ; python_version >= "3.9" and python_version < "4.0"
+lxml-stubs==0.5.1 ; python_version >= "3.9" and python_version < "4.0"
+lxml==5.1.0 ; python_version >= "3.9" and python_version < "4.0"
+mypy-extensions==1.0.0 ; python_version >= "3.9" and python_version < "4.0"
+mypy==1.9.0 ; python_version >= "3.9" and python_version < "4.0"
+nltk==3.8.1 ; python_version >= "3.9" and python_version < "4.0"
+nodeenv==1.8.0 ; python_version >= "3.9" and python_version < "4.0"
+packaging==24.0 ; python_version >= "3.9" and python_version < "4.0"
+pillow==10.2.0 ; python_version >= "3.9" and python_version < "4.0"
+platformdirs==4.2.0 ; python_version >= "3.9" and python_version < "4.0"
+pluggy==1.4.0 ; python_version >= "3.9" and python_version < "4.0"
+pre-commit==3.5.0 ; python_version >= "3.9" and python_version < "4.0"
+pytest==8.1.1 ; python_version >= "3.9" and python_version < "4.0"
+python-dateutil==2.9.0.post0 ; python_version >= "3.9" and python_version < "4.0"
+pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "4.0"
+regex==2023.12.25 ; python_version >= "3.9" and python_version < "4.0"
+requests-file==2.0.0 ; python_version >= "3.9" and python_version < "4.0"
+requests==2.32.3 ; python_version >= "3.9" and python_version < "4.0"
+ruff==0.3.3 ; python_version >= "3.9" and python_version < "4.0"
+selectolax==0.3.27 ; python_version >= "3.9" and python_version < "4.0"
+setuptools==69.2.0 ; python_version >= "3.9" and python_version < "4.0"
+sgmllib3k==1.0.0 ; python_version >= "3.9" and python_version < "4.0"
+six==1.16.0 ; python_version >= "3.9" and python_version < "4.0"
+soupsieve==2.5 ; python_version >= "3.9" and python_version < "4.0"
+tldextract==5.1.1 ; python_version >= "3.9" and python_version < "4.0"
+tomli==2.0.1 ; python_version >= "3.9" and python_version < "3.11"
+tqdm==4.66.2 ; python_version >= "3.9" and python_version < "4.0"
+types-beautifulsoup4==4.12.0.20240229 ; python_version >= "3.9" and python_version < "4.0"
+types-html5lib==1.1.11.20240228 ; python_version >= "3.9" and python_version < "4.0"
+types-pillow==10.2.0.20240311 ; python_version >= "3.9" and python_version < "4.0"
+types-python-dateutil==2.9.0.20240316 ; python_version >= "3.9" and python_version < "4.0"
+types-requests==2.31.0.6 ; python_version >= "3.9" and python_version < "4.0"
+types-urllib3==1.26.25.14 ; python_version >= "3.9" and python_version < "4.0"
+typing-extensions==4.10.0 ; python_version >= "3.9" and python_version < "4.0"
+urllib3==1.26.18 ; python_version >= "3.9" and python_version < "4.0"
+virtualenv==20.25.1 ; python_version >= "3.9" and python_version < "4.0"

setup.py

@@ -24,8 +24,8 @@
     os.system("python3 setup.py sdist upload -r pypi")
     sys.exit()
 
-if sys.version_info < (3, 8):
-    sys.exit("Sorry, Python < 3.8 is not supported")
+if sys.version_info < (3, 9):
+    sys.exit("Sorry, Python < 3.9 is not supported")
 
 
 with open("requirements.txt", encoding="utf-8") as f:
@@ -45,7 +45,7 @@
     author_email="[email protected]",
     url="https://github.com/AndyTheFactory/newspaper4k",
     packages=packages,
-    python_requires=">=3.8",
+    python_requires=">=3.9",
     include_package_data=True,
     install_requires=required_packages,
     license="MIT",