Make fetching and saving license references work

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

src/attributecode/api.py

@@ -20,6 +20,8 @@
 
 import json
 
+import click
+
 from attributecode import ERROR
 from attributecode import Error
 from attributecode import util
@@ -81,6 +83,7 @@ def request_license_data(api_url, api_key, license_key):
         response_content = response.read().decode('utf-8')
         # FIXME: this should be an ordered dict
         license_data = json.loads(response_content)
+
         if not license_data['results']:
             msg = u"Invalid 'license': %s" % license_key
             errors.append(Error(ERROR, msg))
@@ -127,26 +130,25 @@ def get_license_details(api_url, api_key, license_key):
     return lic, errors
 
 
-def fetch_licenses(abouts, api_url, api_key):
+def fetch_licenses(abouts, api_url, api_key, verbose=False):
     """
     Return a mapping of {license key: License object} given an `abouts` list of
     About object and a list of Error.
     """
     dje_domain = '{uri.scheme}://{uri.netloc}/'.format(uri=urlparse(api_url))
     dje_license_url = urljoin(dje_domain, 'urn/?urn=urn:dje:license:{license_key}')
 
-    licenses_by_key = {}
     errors = []
 
     if have_network_connection():
         if not valid_api_url(api_url):
-            msg = "URL not reachable. Invalid '--api_url'. License generation is skipped."
+            msg = "URL not reachable. Invalid '--api_url'. License retrieval is skipped."
             errors.append(Error(ERROR, msg))
     else:
-        msg = 'Network problem. Please check your Internet connection. License generation is skipped.'
+        msg = 'Network problem. Please check your Internet connection. License retrieval is skipped.'
         errors.append(Error(ERROR, msg))
 
-    msg = "Authorization denied. Invalid '--api_key'. License generation is skipped."
+    msg = "Authorization denied. Invalid '--api_key'. License retrieval is skipped."
     auth_error = Error(ERROR, msg)
 
     # collect unique license keys
@@ -160,18 +162,21 @@ def fetch_licenses(abouts, api_url, api_key):
             about.license_expression, unique=True, simple=True)
         license_keys.update(about_keys)
 
+    licenses_by_key = {}
+
     # fetch license key proper
-    for license_key in license_keys:
+    for license_key in sorted(license_keys):
         # No need to go through fetching all the licensesif  we detected invalid '--api_key'
         if auth_error in errors:
             break
-
         license, errs = get_license_details(api_url, api_key, license_key) #NOQA
         errors.extend(errs)
         if license:
             license.url = dje_license_url.format(license_key=license_key)
-            licenses_by_key[license_key] = license_key
+            licenses_by_key[license_key] = license
 
+            if verbose:
+                click.echo('Fetched license: {}'.format(license_key))
 
     return licenses_by_key, util.unique(errors)
 

src/attributecode/cmd.py

@@ -274,22 +274,24 @@ def gen(location, output,reference, quiet, verbose):
 @click.argument('location',
     required=True,
     metavar='LOCATION',
+    callback=partial(validate_extensions, extensions=('.csv',)),
     type=click.Path(
         exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True))
 
 @click.argument('output',
     required=True,
-    callback=partial(validate_extensions, extensions=('.csv',)),
     metavar='OUTPUT',
-    type=click.Path(exists=False, dir_okay=True, file_okay=False, writable=True, resolve_path=True))
+    type=click.Path(exists=True, dir_okay=True, file_okay=False, writable=True, resolve_path=True))
 
 @click.option('--api-key',
     metavar='API-KEY',
+    envvar='DEJACODE_API_KEY',
     type=str,
     help='DejaCode License Library API KEY.')
 
 @click.option('--api-url',
     metavar='API-URL',
+    envvar='DEJACODE_API_URL',
     type=str,
     help='DejaCode License Library API URL.')
 
@@ -321,7 +323,7 @@ def fetch_licenses(location, output, api_key, api_url, quiet, verbose):  # NOQA
 
     errors, abouts = gen.load_inventory(location)
 
-    licenses_by_key, fetch_errors = api.fetch_licenses(abouts, api_url, api_key)
+    licenses_by_key, fetch_errors = api.fetch_licenses(abouts, api_url, api_key, verbose)
     errors.extend(fetch_errors)
 
     for license in licenses_by_key.values():  # NOQA

src/attributecode/gen.py

@@ -89,9 +89,12 @@ def load_inventory(location, base_dir=None):
             errors.append(Error(ERROR, msg))
             continue
         about_file_path = util.to_posix(about_file_path)
-        
+
+        # Ensure there is no absolute directory path
+        about_file_path = about_file_path.strip('/')
+
         segments = about_file_path.split('/')
-        if any(seg !=seg.strip() for seg in segments):
+        if any(seg != seg.strip() for seg in segments):
             msg = (
                 'Invalid "about_file_path": must not end or start with a space. '
                 'Cannot generate .ABOUT file for: "{}"'.format(about_file_path))
@@ -103,7 +106,6 @@ def load_inventory(location, base_dir=None):
             if base_dir:
                 about.location = os.path.join(base_dir, about_file_path)
 
-            print('creating about:' + repr(about))
             abouts.append(about)
         except Exception as e:
             msg = (
@@ -216,29 +218,31 @@ def generate_about_files(inventory_location, target_dir,
         notices_by_name, extra_licenses_by_key = model.load_license_references(reference_dir)
         # we update (and possibly override) existing API-fetched licenses with local ones
         licenses_by_key.update(extra_licenses_by_key)
+        
+
+    # TODO: validate inventory!!!!! to catch error before creating ABOUT files
 
     # update all licenses and notices
     for about in abouts:
+        # fix the location to ensure this is a proper .ABOUT file
+        loc = about.location
+        if not loc.endswith('.ABOUT'):
+            loc = loc.rstrip('\\/').strip() + '.ABOUT'
+        about.location = loc
+
         about.update_licenses(licenses_by_key)
 
         if about.notice_file:
             notice_text = notices_by_name.get(about.notice_file)
             if not notice_text:
                 msg = (
-                    'Empty or missing notice_file. '
-                    'Cannot generate valid .ABOUT file: {}'.format(about.notice_file))
+                    'Cannot generate valid .ABOUT file for: "{}". '
+                    'Empty or missing notice_file: {}'.format(about.location, about.notice_file))
                 errors.append(Error(ERROR, msg))
-                continue
-            about.notice_text = notice_text
-
-    # TODO: validate inventory!!!!! to catch error begore creating ABOUT files
+            else:
+                about.notice_text = notice_text
 
-    # fix the location to ensure this is a proper .ABOUT file
-    for about in abouts:
-        loc = about.location
-        if not loc.endswith('.ABOUT'):
-            loc = loc.rstrip('\\/').strip() + '.ABOUT'
-        about.location = loc
-        about.dump(location=loc, with_files=True)
+        # create the files proper
+        about.dump(location=about.location, with_files=True)
 
     return unique(errors), abouts

src/attributecode/model.py

@@ -34,6 +34,8 @@
 import re
 import traceback
 
+import click
+
 from attributecode.util import python2
 
 if python2:  # pragma: nocover
@@ -128,11 +130,12 @@ def from_dict(cls, data):
 
     def dump(self, target_dir):
         """
-        Write this license as a .yml data file and a text file in `target_dir`.
+        Write this license as a .yml data file and a .LICENSE text file in
+        `target_dir`.
         """
         data_loc = os.path.join(target_dir, self.key + '.yml')
-        with io.open(data_loc, 'wb') as out:
-            data_loc.write(saneyaml.dump(self.to_dict()))
+        with io.open(data_loc, 'w', encoding='utf-8') as out:
+            out.write(saneyaml.dump(self.to_dict()))
 
         if self.text:
             file_loc = self.file_loc(target_dir)
@@ -145,12 +148,13 @@ def load_license_references(reference_dir):
     Return a mapping of notices as {notice_file: notice text} and a mapping of
     {license key: License} loaded from a `reference_dir`.
     In the `reference_dir`, all the files must be UTF-8 encoded text files:
-     - The notice files MUST have a .NOTICE extension.
      - license files must be named after their license key and can consist of:
       - a text file with .LICENSE extension
       - an optional companion .yml YAML data file with extra license data to load
         as a License obejct.
 
+     - The notice files are any file that is not a license file pair.
+
     For instance, we can have the files foo.LICENSE and foo.yml where foo.yml contains:
 
         key: foo
@@ -159,41 +163,32 @@ def load_license_references(reference_dir):
     """
     notices_by_name = {}
     licenses_by_key = {}
-
     ref_files = os.listdir(reference_dir)
-    for name in ref_files:
-        loc = os.path.join(reference_dir, name)
-
-        if os.path.isdir(loc):
-            # TODO: raise some error?
-            continue
-
-        if name.endswith('.NOTICE'):
-            with io.open(loc, encoding='utf-8') as inp:
-                text = inp.read()
-            notices_by_name[name] = text
-
-
-        if name.endswith('.yml'):
-            loaded = License.load(loc)
-
-            license_key = name.replace('.yml', '')
-            lic = licenses_by_key.get(license_key)
-
-            if lic:
-                lic.update(loaded)
-            else:
-                licenses_by_key[license_key] = loaded
-
-        if name.endswith('.LICENSE'):
-            license_key = name.replace('.LICENSE', '')
-            lic = licenses_by_key.get(license_key) or License(license_key)
-            licenses_by_key[license_key] = lic
-
-            with io.open(loc, encoding='utf-8') as inp:
+    data_files =[f for f in ref_files if f.endswith('.yml')]
+    text_files = set([f for f in ref_files if not f.endswith('.yml')])
+
+    for data_file in data_files:
+        loc = os.path.join(reference_dir, data_file)
+        lic =  License.load(loc)
+        licenses_by_key[lic.key] = lic
+
+        if lic.file not in text_files:
+            click.echo(
+                'WARNING: The reference license: {} does not have a '
+                'corresponding text file: {}'.format(lic.key, lic.file))
+        else:
+            with io.open(lic.file_loc(reference_dir), encoding='utf-8') as inp:
                 text = inp.read()
                 lic.text = text
 
+            text_files.remove(lic.file)
+    
+    # whatever is left are "notice" files
+    for notice_file in text_files:
+        with io.open(loc, encoding='utf-8') as inp:
+            text = inp.read()
+            notices_by_name[notice_file] = text
+
     return notices_by_name, licenses_by_key
 
 

tests/test_api.py

@@ -96,7 +96,7 @@ def test_fetch_licenses(self, have_network_connection, valid_api_url):
         valid_api_url.return_value = False
         error_msg = (
             'Network problem. Please check your Internet connection. '
-            'License generation is skipped.')
+            'License retrieval is skipped.')
         expected = ({}, [Error(ERROR, error_msg)])
         assert api.fetch_licenses([], '', '') == expected