Skip to content

280 python3 cleanup #301

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chinyeungli merged 7 commits into from
Nov 17, 2017
Merged

280 python3 cleanup #301

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
77c33a4
Fix Python2/Python3 compatibility for the api module #280
tdruez Nov 15, 2017
eec9807
Fix Python2/Python3 compatibility for the `model` module #280
tdruez Nov 15, 2017
a0d74fd
Remove un-used imports #280
tdruez Nov 15, 2017
13a5fe8
Rename command function names to avoid conflict with module names #280
tdruez Nov 15, 2017
e186aa5
Refine import to prevent name conflict #280
tdruez Nov 15, 2017
a6c74ea
Decode response_content before loading in JSON #297
tdruez Nov 17, 2017
18bc355
Add FetchLicenseTest for basic test coverage #280
tdruez Nov 17, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 23 additions & 102 deletions src/attributecode/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,25 +18,15 @@
from __future__ import print_function
from __future__ import unicode_literals

from collections import namedtuple
import json

try:
from urllib import urlencode
from urllib import quote
except ImportError:
from urllib.parse import urlencode
from urllib.parse import quote

try:
import httplib # Python 2
except ImportError:
import http.client as httplib # Python 3

try:
import urllib2 # Python 2
except ImportError:
import urllib as urllib2 # Python 3
try: # Python 2
from urllib import urlencode, quote
from urllib2 import urlopen, Request, HTTPError, URLError
except ImportError: # Python 3
from urllib.parse import urlencode, quote
from urllib.request import urlopen, Request
from urllib.error import HTTPError, URLError

from attributecode import ERROR
from attributecode import Error
Expand All @@ -46,87 +36,16 @@
API call helpers
"""

def build_api_url(url, api_username, api_key, license_key):
"""
Return a URl suitable for making an API call.
"""
url = url.rstrip('/')

payload = {'username': api_username,
'api_key': api_key,
'format': 'json'}
encoded_payload = urlencode(payload)

api_url = '%(url)s/%(license_key)s/?%(encoded_payload)s' % locals()

# handle special characters in URL such as space etc.
api_url = quote(api_url, safe="%/:=&?~#+!$,;'@()*[]")
return api_url


def get_license_data(self, url, api_username, api_key, license_key):
"""
Return a list of errors and a dictionary of license data, given a DejaCode
API url and a DejaCode license_key, send an API request to get license
data for the license_key, authenticating through an api_key and username.
"""
full_url = build_api_url(url, api_username, api_key, license_key)

errors = []
license_data = {}

msg = 'Failed to collect license data for %(license_key)s.'

try:
request = urllib2.Request(full_url)
response = urllib2.urlopen(request)
response_content = response.read()
license_data = json.loads(response_content)

except urllib2.HTTPError as e:
if e.code == httplib.UNAUTHORIZED:
msg = msg + ('Authorization denied: '
'Invalid api_username: %(api_username)s '
'or api_key: %(api_key)s.')
# FIXME: what about 404 and other cases?
errors.append(Error(ERROR, msg % locals()))

except urllib2.URLError as e:
msg = msg + 'Network problem. Check your internet connection.'
errors.append(Error(ERROR, msg % locals()))

except Exception as e:
# only keep the first 100 char of the exception
emsg = repr(e)[:100]
msg = msg + ' Error: %(emsg)s'
errors.append(Error(ERROR, msg % locals()))

return errors, license_data


LicenseInfo = namedtuple('LicenseInfo', ['key', 'name', 'text'])


def get_license_info(self, url, api_username, api_key, license_key,
caller=get_license_data):
"""
Return a list of errors and a tuple of key, name, text for a given
license_key using a DejaCode API request at url. caller is the function
used to call the API and is used mostly for mocking in tests.
"""
errors, data = get_license_data(url, api_username, api_key, license_key)
key = data.get('key')
name = data.get('name')
text = data.get('full_text')
return errors, LicenseInfo(key, name, text)


def request_license_data(url, api_key, license_key):
"""
Return a dictionary of license data.
Send a request to a given API URL to gather license data for
license_key, authenticating through an api_key.
"""
headers = {
'Authorization': 'Token %s' % api_key,
}
payload = {
'api_key': api_key,
'key': license_key,
Expand All @@ -137,40 +56,42 @@ def request_license_data(url, api_key, license_key):
encoded_payload = urlencode(payload)
full_url = '%(url)s/?%(encoded_payload)s' % locals()
# handle special characters in URL such as space etc.
full_url = quote(full_url, safe="%/:=&?~#+!$,;'@()*[]")
headers = {'Authorization': 'Token %s' % api_key}
quoted_url = quote(full_url, safe="%/:=&?~#+!$,;'@()*[]")

license_data = {}
errors = []
try:
request = urllib2.Request(full_url, headers=headers)
response = urllib2.urlopen(request)
response_content = response.read()
request = Request(quoted_url, headers=headers)
response = urlopen(request)
response_content = response.read().decode('utf-8')
license_data = json.loads(response_content)
if not license_data['results']:
msg = (u"Invalid 'license': " + license_key)
msg = u"Invalid 'license': %s" % license_key
errors.append(Error(ERROR, msg))
except urllib2.HTTPError as http_e:
except HTTPError as http_e:
# some auth problem
if http_e.code == 403:
msg = (u"Authorization denied. Invalid '--api_key'. License generation is skipped.")
msg = (u"Authorization denied. Invalid '--api_key'. "
u"License generation is skipped.")
errors.append(Error(ERROR, msg))
else:
# Since no api_url/api_key/network status have
# problem detected, it yields 'license' is the cause of
# this exception.
msg = (u"Invalid 'license': " + license_key)
msg = u"Invalid 'license': %s" % license_key
errors.append(Error(ERROR, msg))
except Exception as e:
errors.append(Error(ERROR, str(e)))
finally:
license_data = license_data.get('results')[0] if license_data.get('count') == 1 else {}

return license_data, errors


def get_license_details_from_api(url, api_key, license_key):
"""
Returns the license_text of a given license_key using an API request.
Returns an empty string if the text is not available.
Return the license_text of a given license_key using an API request.
Return an empty string if the text is not available.
"""
license_data, errors = request_license_data(url, api_key, license_key)
license_name = license_data.get('name', '')
Expand Down
21 changes: 6 additions & 15 deletions src/attributecode/cmd.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
from __future__ import print_function
from __future__ import unicode_literals

import codecs
import logging
import os
from os.path import exists, join
Expand All @@ -28,19 +27,11 @@
# silence unicode literals warnings
click.disable_unicode_literals_warning = True

import attributecode
from attributecode import CRITICAL
from attributecode import ERROR
from attributecode import INFO
from attributecode import NOTSET
from attributecode import WARNING
from attributecode import __about_spec_version__
from attributecode import __version__
from attributecode import attrib
from attributecode import Error
from attributecode import gen
from attributecode.attrib import generate_and_save as attrib_generate_and_save
from attributecode.gen import generate as gen_generate
from attributecode import model
from attributecode.model import About
from attributecode import severities
from attributecode.util import extract_zip
from attributecode.util import to_posix
Expand Down Expand Up @@ -147,7 +138,7 @@ def inventory(location, output, quiet, format):
# accept zipped ABOUT files as input
location = extract_zip(location)

errors, abouts = attributecode.model.collect_inventory(location)
errors, abouts = model.collect_inventory(location)

write_errors = model.write_output(abouts, output, format)
for err in write_errors:
Expand Down Expand Up @@ -209,7 +200,7 @@ def gen(location, output, mapping, license_notice_text_location, fetch_license,

click.echo('Generating .ABOUT files...')

errors, abouts = attributecode.gen.generate(
errors, abouts = gen_generate(
location=location, base_dir=output, use_mapping=mapping,
license_notice_text_location=license_notice_text_location,
fetch_license=fetch_license)
Expand Down Expand Up @@ -274,7 +265,7 @@ def attrib(location, output, template, mapping, inventory, quiet):
location = extract_zip(location)

inv_errors, abouts = model.collect_inventory(location)
no_match_errors = attributecode.attrib.generate_and_save(
no_match_errors = attrib_generate_and_save(
abouts=abouts, output_location=output,
use_mapping=mapping, template_loc=template,
inventory_location=inventory)
Expand Down Expand Up @@ -316,7 +307,7 @@ def check(location, show_all):
click.echo('Running aboutcode-toolkit version ' + __version__)
click.echo('Checking ABOUT files...')

errors, abouts = attributecode.model.collect_inventory(location)
errors, abouts = model.collect_inventory(location)

msg_format = '%(sever)s: %(message)s'
print_errors = []
Expand Down
Loading