[Refactor] Download component

gogdl/__init__.py

@@ -1,12 +1,10 @@
-"""
- _                    _                                 _ _
-| |__   ___ _ __ ___ (_) ___       __ _  ___   __ _  __| | |
-| '_ \ / _ \ '__/ _ \| |/ __|____ / _` |/ _ \ / _` |/ _` | |
-| | | |  __/ | | (_) | | (_|_____| (_| | (_) | (_| | (_| | |
-|_| |_|\___|_|  \___/|_|\___|     \__, |\___/ \__, |\__,_|_|
-                                  |___/       |___/
+#  _                    _                                 _ _
+# | |__   ___ _ __ ___ (_) ___       __ _  ___   __ _  __| | |
+# | '_ \ / _ \ '__/ _ \| |/ __|____ / _` |/ _ \ / _` |/ _` | |
+# | | | |  __/ | | (_) | | (_|_____| (_| | (_) | (_| | (_| | |
+# |_| |_|\___|_|  \___/|_|\___|     \__, |\___/ \__, |\__,_|_|
 
-"""
 
 
-version = "0.7.3"
+
+version = "1.0.0"

gogdl/api.py

@@ -6,6 +6,8 @@
 from gogdl.dl import dl_utils
 from gogdl import version
 import gogdl.constants as constants
+from gogdl import version
+
 
 
 class ApiHandler:
@@ -27,7 +29,9 @@ def __init__(self, auth_manager):
         self.endpoints = dict()  # Map of secure link endpoints
         self.working_on_ids = list()  # List of products we are waiting for to complete getting the secure link
 
-    def get_item_data(self, id, expanded=[]):
+    def get_item_data(self, id, expanded=None):
+        if expanded is None:
+            expanded = []
         self.logger.info(f"Getting info from products endpoint for id: {id}")
         url = f'{constants.GOG_API}/products/{id}'
         expanded_arg = '?expand='
@@ -48,17 +52,15 @@ def get_game_details(self, id):
         if response.ok:
             return response.json()
 
-    def get_dependenices_list(self, depot_version=2):
+    def get_dependencies_repo(self, depot_version=2):
         self.logger.info("Getting Dependencies repository")
         url = constants.DEPENDENCIES_URL if depot_version == 2 else constants.DEPENDENCIES_V1_URL
         response = self.session.get(url)
         if not response.ok:
             return None
 
         json_data = json.loads(response.content)
-        if 'repository_manifest' in json_data:
-            self.logger.info("Getting repository manifest")
-            return dl_utils.get_zlib_encoded(self, str(json_data['repository_manifest']), self.logger)[0], json_data.get('version')
+        return json_data
 
     def does_user_own(self, id):
         if not self.owned:

gogdl/args.py

@@ -1,5 +1,6 @@
 # Initialize argparse module and return arguments
 import argparse
+from multiprocessing import cpu_count
 
 
 def init_parser():
@@ -20,23 +21,49 @@ def init_parser():
 
     subparsers = parser.add_subparsers(dest="command")
 
+    import_parser = subparsers.add_parser(
+        "import", help="Show data about game in the specified path"
+    )
+    import_parser.add_argument("path")
+
+    # REDIST DOWNLOAD
+
+    redist_download_parser = subparsers.add_parser("redist", aliases=["dependencies"],
+                                                   help="Download specified dependencies to provided location")
+
+    redist_download_parser.add_argument("--ids", help="Coma separated ids")
+    redist_download_parser.add_argument("--path", help="Location where to download the files")
+    redist_download_parser.add_argument("--print-manifest", action="store_true", help="Prints manifest to stdout and exits")
+    redist_download_parser.add_argument(
+        "--max-workers",
+        dest="workers_count",
+        default=cpu_count(),
+        help="Specify number of worker threads, by default number of CPU threads",
+    )
+
+
+    # AUTH
+
     auth_parser = subparsers.add_parser("auth", help="Manage authorization")
     auth_parser.add_argument("--client-id", dest="client_id")
     auth_parser.add_argument("--client-secret", dest="client_secret")
     auth_parser.add_argument("--code", dest="authorization_code",
                              help="Pass authorization code (use for login), when passed client-id and secret are ignored")
 
+    # DOWNLOAD
+
     download_parser = subparsers.add_parser(
         "download", aliases=["repair", "update"], help="Download/update/repair game"
     )
     download_parser.add_argument("id", help="Game id")
     download_parser.add_argument("--lang", "-l", help="Specify game language")
     download_parser.add_argument(
-        "--build", "-b", dest="build", help="Specify buildId (allows repairing)"
+        "--build", "-b", dest="build", help="Specify buildId"
     )
     download_parser.add_argument(
         "--path", "-p", dest="path", help="Specify download path", required=True
     )
+    download_parser.add_argument("--support", dest="support_path", help="Specify path where support files should be stored, by default they are put into game dir")
     download_parser.add_argument(
         "--platform",
         "--os",
@@ -45,26 +72,53 @@ def init_parser():
         choices=["windows", "osx", "linux"],
     )
     download_parser.add_argument(
-        "--with-dlcs", dest="dlcs", action="store_true", help="Should download dlcs"
+        "--with-dlcs", dest="dlcs", action="store_true", help="Should download all dlcs"
+    )
+    download_parser.add_argument(
+        "--skip-dlcs", dest="dlcs", action="store_false", help="Should skip all dlcs"
+    )
+    download_parser.add_argument(
+        "--dlcs",
+        dest="dlcs_list",
+        default=[],
+        help="List of dlc ids to download (separated by coma)",
     )
     download_parser.add_argument(
-        "--skip-dlcs", dest="dlcs", action="store_false", help="Should skip dlcs"
+        "--dlc-only", dest="dlc_only", action="store_true", help="Download only DLC"
     )
+    download_parser.add_argument("--branch", help="Choose build branch to use")
+    download_parser.add_argument("--password", help="Password to access other branches")
+    download_parser.add_argument("--force-gen", choices=["1", "2"], dest="force_generation", help="Force specific manifest generation (FOR DEBUGGING)")
     download_parser.add_argument(
         "--max-workers",
         dest="workers_count",
-        default=0,
+        default=cpu_count(),
         help="Specify number of worker threads, by default number of CPU threads",
     )
 
-    import_parser = subparsers.add_parser(
-        "import", help="Show data about game in the specified path"
-    )
-    import_parser.add_argument("path")
+    # SIZE CALCULATING, AND OTHER MANIFEST INFO
 
     calculate_size_parser = subparsers.add_parser(
         "info", help="Calculates estimated download size and list of DLCs"
     )
+
+    calculate_size_parser.add_argument(
+        "--with-dlcs",
+        dest="dlcs",
+        action="store_true",
+        help="Should download all dlcs",
+    )
+    calculate_size_parser.add_argument(
+        "--skip-dlcs", dest="dlcs", action="store_false", help="Should skip all dlcs"
+    )
+    calculate_size_parser.add_argument(
+        "--dlcs",
+        dest="dlcs_list",
+        help="Coma separated list of dlc ids to download",
+    )
+    calculate_size_parser.add_argument(
+        "--dlc-only", dest="dlc_only", action="store_true", help="Download only DLC"
+    )
     calculate_size_parser.add_argument("id")
     calculate_size_parser.add_argument(
         "--platform",
@@ -77,13 +131,18 @@ def init_parser():
         "--build", "-b", dest="build", help="Specify buildId"
     )
     calculate_size_parser.add_argument("--lang", "-l", help="Specify game language")
+    calculate_size_parser.add_argument("--branch", help="Choose build branch to use")
+    calculate_size_parser.add_argument("--password", help="Password to access other branches")
+    calculate_size_parser.add_argument("--force-gen", choices=["1", "2"], dest="force_generation", help="Force specific manifest generation (FOR DEBUGGING)")
     calculate_size_parser.add_argument(
         "--max-workers",
         dest="workers_count",
-        default=0,
+        default=cpu_count(),
         help="Specify number of worker threads, by default number of CPU threads",
     )
 
+    # LAUNCH
+
     launch_parser = subparsers.add_parser(
         "launch", help="Launch the game in specified path", add_help=False
     )
@@ -108,6 +167,8 @@ def init_parser():
         "--override-exe", dest="override_exe", help="Override executable to be run"
     )
 
+    # SAVES
+
     save_parser = subparsers.add_parser("save-sync", help="Sync game saves")
     save_parser.add_argument("path", help="Path to sync files")
     save_parser.add_argument("id", help="Game id")
@@ -143,6 +204,8 @@ def init_parser():
         required=True,
     )
 
+    # SAVES CLEAR
+
     clear_parser = subparsers.add_parser("save-clear", help="Clear cloud game saves")
     clear_parser.add_argument("path", help="Path to sync files")
     clear_parser.add_argument("id", help="Game id")
@@ -157,4 +220,5 @@ def init_parser():
         required=True,
     )
 
+
     return parser.parse_known_args()

gogdl/cli.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
+from multiprocessing import freeze_support
 import gogdl.args as args
-from gogdl.dl import manager
+from gogdl.dl.managers import manager
+from gogdl.dl.managers import dependencies
 import gogdl.api as api
 import gogdl.imports as imports
 import gogdl.launch as launch
@@ -9,8 +11,6 @@
 from gogdl import version as gogdl_version
 import logging
 
-logging.basicConfig(format="[%(name)s] %(levelname)s: %(message)s", level=logging.INFO)
-logger = logging.getLogger("MAIN")
 
 
 def display_version():
@@ -19,6 +19,11 @@ def display_version():
 
 def main():
     arguments, unknown_args = args.init_parser()
+    level = logging.INFO
+    if '-d' in unknown_args or '--debug' in unknown_args:
+        level = logging.DEBUG
+    logging.basicConfig(format="[%(name)s] %(levelname)s: %(message)s", level=level)
+    logger = logging.getLogger("MAIN")
     logger.debug(arguments)
     if arguments.display_version:
         display_version()
@@ -28,24 +33,35 @@ def main():
         return
     authorization_manager = auth.AuthorizationManager(arguments.auth_config_path)
     api_handler = api.ApiHandler(authorization_manager)
-    download_manager = manager.DownloadManager(api_handler)
     clouds_storage_manager = saves.CloudStorageManager(api_handler, authorization_manager)
-    switcher = {
-        "download": download_manager.download,
-        "repair": download_manager.download,
-        "update": download_manager.download,
-        "import": imports.get_info,
-        "info": download_manager.calculate_download_size,
-        "launch": launch.launch,
-        "save-sync": clouds_storage_manager.sync,
-        "save-clear": clouds_storage_manager.clear,
-        "auth": authorization_manager.handle_cli
-    }
+
+    switcher = {}
+    if arguments.command in ["download", "repair", "update", "info"]:
+        download_manager = manager.Manager(arguments, unknown_args, api_handler)
+        switcher = {
+            "download": download_manager.download,
+            "repair": download_manager.download,
+            "update": download_manager.download,
+            "info": download_manager.calculate_download_size,
+        }
+    elif arguments.command in ["redist", "dependencies"]:
+        dependencies_handler = dependencies.DependenciesManager(arguments.ids.split(","), arguments.path, arguments.workers_count, api_handler, print_manifest=arguments.print_manifest)
+        if not arguments.print_manifest:
+            dependencies_handler.get()
+    else:
+        switcher = {
+            "import": imports.get_info,
+            "launch": launch.launch,
+            "save-sync": clouds_storage_manager.sync,
+            "save-clear": clouds_storage_manager.clear,
+            "auth": authorization_manager.handle_cli
+        }
 
     function = switcher.get(arguments.command)
     if function:
         function(arguments, unknown_args)
 
 
 if __name__ == "__main__":
+    freeze_support()
     main()

gogdl/constants.py

@@ -10,13 +10,22 @@
 DEPENDENCIES_URL = "https://content-system.gog.com/dependencies/repository?generation=2"
 DEPENDENCIES_V1_URL = "https://content-system.gog.com/redists/repository"
 
+NON_NATIVE_SEP = "\\" if os.sep == "/" else "/"
 
-# Use only for Linux
-CACHE_DIR = (
-    os.path.join(
-        os.getenv("XDG_CACHE_HOME", os.path.join(os.path.expanduser("~"), ".cache")),
-        "heroicGOGdl",
+if platform == 'linux':
+    CONFIG_DIR = os.path.join(
+        os.getenv("XDG_CONFIG_HOME", os.path.expanduser("~/.config")), 'heroic_gogdl'
     )
-    if platform == "linux"
-    else ""
-)
+elif platform == 'win32':
+    CONFIG_DIR = os.path.join(
+        os.getenv("APPDATA"), 'heroic_gogdl'
+    )
+elif platform == 'darwin':
+    CONFIG_DIR = os.path.join(
+        os.path.expanduser("~/Library"), "Application Support", "heroic_gogdl"
+    )
+
+if os.getenv("GOGDL_CONFIG_PATH"):
+    CONFIG_DIR = os.path.join(os.getenv("GOGDL_CONFIG_PATH"), "heroic_gogdl")
+
+MANIFESTS_DIR = os.path.join(CONFIG_DIR, "manifests")