Add LIS, validate/modify all existing

api/admin.py

@@ -29,7 +29,7 @@ class ETLDatasetAdmin(admin.ModelAdmin):
     formfield_overrides = {
         models.JSONField: {'widget': JSONEditorWidget}
     }
-    list_display = ('dataset_name','dataset_subtype')
+    list_display = ('dataset_name','dataset_subtype', 'is_pipeline_enabled', 'dataset_availability')
 
 @admin.register(Profile)
 class ProfileAdmin(admin.ModelAdmin):

api/etl/daily_iterator.py

@@ -14,6 +14,7 @@ def __init__(self, start_date, end_date):
     def get_range(self):
         delta = self.end_date - self.start_date
         date_range = [self.start_date + datetime.timedelta(days=x) for x in range(delta.days + 1)]
-        date_range_string = [date.strftime('%Y%m%d') for date in date_range]
 
-        return date_range_string
+        return_obj = [{'date': date.strftime('%Y%m%d')} for date in date_range]
+
+        return return_obj

api/etl/dekadal_iterator.py

@@ -32,7 +32,8 @@ def get_range(self):
         for year in range(start_year, end_year + 1):
             for dekad in range(start_dekad if year == start_year else 1, end_dekad + 1 if year == end_year else 36):
                 date_range.append(self.get_begin_date(dekad, year))
-                date_range_string = [date.strftime('%Y%m%d') for date in date_range]
 
-        return date_range_string
+        return_obj = [{'date': date.strftime('%Y%m%d')} for date in date_range]
+
+        return return_obj
 

api/etl/esi_iterator.py

@@ -0,0 +1,41 @@
+import datetime
+import requests
+import re
+
+from bs4 import BeautifulSoup
+
+
+class ESI_Iterator:
+    start_date = None
+    end_date = None
+    root_url = None
+    date_format = "YYYYMMDD"
+
+    # Default Constructor using datetime date
+    def __init__(self, start_date, end_date, options):
+        self.start_date = start_date
+        self.end_date = end_date
+        self.root_url = options['root_url']
+        self.week_type = options['week_type']
+        self.delta = 28 if self.week_type == '4WK' else 84
+
+    def get_range(self):
+        return_obj = []
+        response = requests.get(self.root_url)
+        soup = BeautifulSoup(response.text, 'html.parser')
+        for _, link in enumerate(soup.findAll('a')):
+            if link.get('href').endswith('.tif.gz'):
+                _, wk, rest = link.get('href').split('_')
+                if wk == self.week_type:
+                    end_time = datetime.datetime.strptime('{} {}'.format(rest[4:].replace('.tif.gz', ''), rest[:4]),
+                                                      '%j %Y')
+                    date = end_time - datetime.timedelta(days=self.delta)
+                    filename = link.get_text()
+                    if self.start_date <= date.date() <= self.end_date:
+                        return_obj.append({
+                            'date': date.strftime('%Y%m%d'),
+                            'filename': filename,
+                            'source': self.root_url + filename,
+                        })
+
+        return return_obj