#
# (C) Copyright 2012-2013 ECMWF.
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
# In applying this licence, ECMWF does not waive the privileges and immunities
# granted to it by virtue of its status as an intergovernmental organisation nor
# does it submit to any jurisdiction.
# make the python3-like print behave in python 2
from __future__ import print_function
import os
import sys
import time
import traceback
from contextlib import closing
# python 2 and 3 compatible urllib and httplib imports
try:
from urllib.parse import urlparse
from urllib.parse import urljoin
from urllib.error import HTTPError, URLError
from urllib.request import HTTPRedirectHandler, Request, build_opener, urlopen, addinfourl
from http.client import BadStatusLine
except ImportError:
from urlparse import urlparse
from urlparse import urljoin
from urllib2 import HTTPError, URLError
from urllib2 import HTTPRedirectHandler, Request, build_opener, urlopen, addinfourl
from httplib import BadStatusLine
try:
import json
except ImportError:
import simplejson as json
try:
import ssl
except ImportError:
print("Python socket module was not compiled with SSL support. Aborting...")
sys.exit(1)
###############################################################################
VERSION = '1.5.0'
###############################################################################
class APIKeyFetchError(Exception):
pass
def _get_apikey_from_environ():
try:
key = os.environ["ECMWF_API_KEY"]
url = os.environ["ECMWF_API_URL"]
email = os.environ["ECMWF_API_EMAIL"]
return key, url, email
except KeyError:
raise APIKeyFetchError("ERROR: Could not get the API key from the environment")
def _get_apikey_from_rcfile():
rc = os.path.normpath(os.path.expanduser("~/.ecmwfapirc"))
try:
with open(rc) as f:
config = json.load(f)
except IOError as e: # Failed reading from file
raise APIKeyFetchError(str(e))
except ValueError: # JSON decoding failed
raise APIKeyFetchError("ERROR: Missing or malformed API key in '%s'" % rc)
except Exception as e: # Unexpected error
raise APIKeyFetchError(str(e))
try:
key = config["key"]
url = config["url"]
email = config["email"]
return key, url, email
except:
raise APIKeyFetchError("ERROR: Missing or malformed API key in '%s'" % rc)
def get_apikey_values():
"""Get the API key from the environment or the '.ecmwfapirc' file.
The environment is looked at first.
Returns:
Tuple with the key, url, and email forming our API key.
Raises:
APIKeyFetchError: When unable to get the API key from either the
environment or the ecmwfapirc file.
"""
try:
key_values = _get_apikey_from_environ()
except APIKeyFetchError:
try:
key_values = _get_apikey_from_rcfile()
except APIKeyFetchError:
raise
return key_values
###############################################################################
class RetryError(Exception):
def __init__(self, code, text):
self.code = code
self.text = text
def __str__(self):
return "%d %s" % (self.code, self.text)
class APIException(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
def robust(func):
def wrapped(self, *args, **kwargs):
max_tries = tries = 10
delay = 60 # retry delay
last_error = None
while tries > 0:
try:
return func(self, *args, **kwargs)
except HTTPError as e:
if self.verbose:
print("WARNING: HTTPError received %s" % (e))
if e.code < 500 or e.code in (501,): # 501: not implemented
raise
last_error = e
except BadStatusLine as e:
if self.verbose:
print("WARNING: BadStatusLine received %s" % (e))
last_error = e
except URLError as e:
if self.verbose:
print("WARNING: URLError received %s %s" % (e.errno, e))
last_error = e
except APIException:
raise
except RetryError as e:
if self.verbose:
print("WARNING: HTTP received %s" % (e.code))
print(e.text)
last_error = e
except:
if self.verbose:
print("Unexpected error:", sys.exc_info()[0])
print(traceback.format_exc())
raise
print("Error contacting the WebAPI, retrying in %d seconds ..." % delay)
time.sleep(delay)
tries -= 1
# if all retries have been exhausted, raise the last exception caught
print("Could not contact the WebAPI after %d tries, failing !" % max_tries)
raise last_error
return wrapped
def get_api_url(url):
parsed_uri = urlparse(url)
return '{uri.scheme}://{uri.netloc}/{apiver}/'.format(
uri=parsed_uri, apiver=parsed_uri.path.split('/')[1])
SAY = True
class Ignore303(HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
if code in [301, 302]:
# We want the posts to work even if we are redirected
if code == 301:
global SAY
if SAY:
o = req.get_full_url()
n = newurl
print()
print("*** ECMWF API has moved")
print("*** OLD: %s" % get_api_url(o))
print("*** NEW: %s" % get_api_url(n))
print("*** Please update your ~/.ecmwfapirc file")
print()
SAY = False
try:
# Python < 3.4
data = req.get_data()
except AttributeError:
# Python >= 3.4
data = req.data
try:
# Python < 3.4
origin_req_host = req.get_origin_req_host()
except AttributeError:
# Python >= 3.4
origin_req_host = req.origin_req_host
return Request(newurl,
data=data,
headers=req.headers,
origin_req_host=origin_req_host,
unverifiable=True)
return None
def http_error_303(self, req, fp, code, msg, headers):
infourl = addinfourl(fp, headers, req.get_full_url())
infourl.status = code
infourl.code = code
return infourl
class Connection(object):
def __init__(self, url, email=None, key=None, verbose=False, quiet=False):
self.url = url
self.email = email
self.key = key
self.retry = 5
self.location = None
self.done = False
self.value = True
self.offset = 0
self.verbose = verbose
self.quiet = quiet
self.status = None
@robust
def call(self, url, payload=None, method="GET"):
# Ensure full url
url = urljoin(self.url, url)
if self.verbose:
print(method, url)
headers = {"Accept": "application/json", "From": self.email, "X-ECMWF-KEY": self.key}
opener = build_opener(Ignore303)
data = None
if payload is not None:
data = json.dumps(payload).encode('utf-8')
headers["Content-Type"] = "application/json"
url = "%s?offset=%d&limit=500" % (url, self.offset)
req = Request(url=url, data=data, headers=headers)
if m
