only allow multiprocess on Linux

datawhores · Aug 30, 2024 · 8e394e8 · 8e394e8
1 parent a959ca8
commit 8e394e8
Show file tree

Hide file tree

Showing 6 changed files with 14 additions and 20 deletions.
diff --git a/ofscraper/__main__.py b/ofscraper/__main__.py
@@ -6,7 +6,6 @@
 
 
 def main():
-    system.set_mulitproc_start_type()
     if system.get_parent():
         load.main()
 

diff --git a/ofscraper/actions/actions/download/batch/downloadbatch.py b/ofscraper/actions/actions/download/batch/downloadbatch.py
@@ -90,10 +90,9 @@ async def process_dicts(username, model_id, filtered_medialist, posts):
                     count=1,
                 )
                 log_threads.append(thread)
-
             processes = [
-                aioprocessing.AioProcess(
-                    target=process_dict_starter,
+                aioprocessing.mp.get_context(system.get_mulitproc_start_type()).Process(
+                    target=process_dict_starter,    
                     args=(
                         username,
                         model_id,

diff --git a/ofscraper/actions/actions/download/download.py b/ofscraper/actions/actions/download/download.py
@@ -58,6 +58,7 @@ async def download_picker(username, model_id, medialist, posts):
             >= settings.get_threads() * constants.getattr("DOWNLOAD_THREAD_MIN")
         )
         and settings.not_solo_thread()
+        and system.platform.system()=="Linux"
     ):
         return await batch.process_dicts(username, model_id, medialist, posts)
     else:

diff --git a/ofscraper/actions/actions/metadata/batch/metadatabatch.py b/ofscraper/actions/actions/metadata/batch/metadatabatch.py
@@ -74,7 +74,7 @@ def process_dicts(username, model_id, filtered_medialist):
                 log_threads.append(thread)
 
             processes = [
-                aioprocessing.AioProcess(
+                aioprocessing.mp.get_context(system.get_mulitproc_start_type()).Process(
                     target=process_dict_starter,
                     args=(
                         username,

diff --git a/ofscraper/actions/actions/metadata/metadata.py b/ofscraper/actions/actions/metadata/metadata.py
@@ -29,6 +29,8 @@ async def metadata_picker(username, model_id, medialist):
             >= settings.get_threads() * constants.getattr("DOWNLOAD_THREAD_MIN")
         )
         and settings.not_solo_thread()
+        and system.platform.system()=="Linux"
+
     ):
         return batch.process_dicts(username, model_id, medialist)
     else:

diff --git a/ofscraper/utils/system/system.py b/ofscraper/utils/system/system.py
@@ -10,6 +10,8 @@
 import multiprocess
 import psutil
 from setproctitle import setproctitle
+import aioprocessing
+
 
 
 def is_frozen():
@@ -47,29 +49,20 @@ def getOpenFiles(unique=True):
     return out
 
 
-def set_mulitproc_start_type():
+def get_mulitproc_start_type():
     plat = platform.system()
     if is_frozen():
         f_method = "spawn"
-        multiprocess.set_start_method(f_method)
-        multiprocessing.set_start_method(f_method)
+        return f_method
     elif plat == "Darwin":
         d_method = "spawn"
-        multiprocess.set_start_method(d_method)
-        multiprocessing.set_start_method(d_method)
+        return d_method
     elif plat == "Windows":
         w_method = "spawn"
-        multiprocess.set_start_method(w_method)
-        multiprocessing.set_start_method(w_method)
+        return w_method
     else:
-        o_method = "forkserver"
-        multiprocess.set_start_method(o_method)
-        multiprocessing.set_start_method(o_method)
-    # additional for mac
-    if plat == "Darwin":
-        os.environ["OBJC_DISABLE_INITIALIZE_FORK_SAFETY"] = "YES"
-        os.environ["no_proxy"] = "*"
-
+        o_method = "spawn"
+        return o_method
 
 def set_eventloop():
     plat = platform.system()