From d5f613d68e874f71211ab91842f97de4cfa235ec Mon Sep 17 00:00:00 2001
From: Brian Thomas
Date: Tue, 8 Dec 2015 11:53:54 -0500
Subject: [PATCH 001/146] update returned example JSON
---
apod/templates/home.html | 15 ++++++---------
1 file changed, 6 insertions(+), 9 deletions(-)
diff --git a/apod/templates/home.html b/apod/templates/home.html
index a414dea..86ac35f 100644
--- a/apod/templates/home.html
+++ b/apod/templates/home.html
@@ -30,7 +30,7 @@ Service API
-curl http://{{ service_url }}/{{ version }}/{{ methodname }}/?concept_tags=True&date=2010-1-1
+curl http://{{ service_url }}/{{ version }}/{{ methodname }}/?concept_tags=True&date=2015-10-11
@@ -40,15 +40,12 @@
Service API
{
- "dictionary": "nasa_opendata_trained_model_09_17_15.pkl",
- "highest_ngram_allowed": 5,
- "keywords": [
- "nasa",
- "rockets"
- ],
+ "concepts": "concept_tags functionality turned off in current service",
+ "date": "2015-10-11",
+ "explanation": "Clouds of glowing gas mingle with dust lanes in the Trifid Nebula, a star forming region toward the constellation of the Archer (Sagittarius). In the center, the three prominent dust lanes that give the Trifid its name all come together. Mountains of opaque dust appear on the right, while other dark filaments of dust are visible threaded throughout the nebula. A single massive star visible near the center causes much of the Trifid's glow. The Trifid, also known as M20, is only about 300,000 years old, making it among the youngest emission nebulae known. The nebula lies about 9,000 light years away and the part pictured here spans about 10 light years. The above image is a composite with luminance taken from an image by the 8.2-m ground-based Subaru Telescope, detail provided by the 2.4-m orbiting Hubble Space Telescope, color data provided by Martin Pugh and image assembly and processing provided by Robert Gendler. Follow APOD on: Facebook, Google Plus, or Twitter",
"service_version": "v1",
- "term_count_threshold": 1,
- "textmining_library_version": "0.5.1"
+ "title": "In the Center of the Trifid Nebula",
+ "url": "http://apod.nasa.gov/apod/image/1510/Trifid_HubbleGendler_960.jpg"
}
From a44a98fe7ac2ddba49da2f04fa9af31ddbc4546a Mon Sep 17 00:00:00 2001
From: Brian Thomas
Date: Tue, 5 Jan 2016 13:48:42 -0500
Subject: [PATCH 002/146] Add CORS for all endpoints
---
apod/service.py | 6 ++++--
requirements.txt | 2 +-
2 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/apod/service.py b/apod/service.py
index c73b2a3..7347476 100644
--- a/apod/service.py
+++ b/apod/service.py
@@ -8,13 +8,15 @@
@author=bathomas @email=brian.a.thomas@nasa.gov
'''
+from bs4 import BeautifulSoup
+from datetime import datetime
from flask import request, jsonify, render_template, Response, Flask
+from flask.ext.cors import CORS
import json
-from datetime import datetime
-from bs4 import BeautifulSoup
import requests
app = Flask(__name__)
+CORS(app)
# this should reflect both this service and the backing
# assorted libraries
diff --git a/requirements.txt b/requirements.txt
index b30e918..5588427 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,7 +4,7 @@
# in `lib/` subdirectory.
#
# Note: The `lib` directory is added to `sys.path` by `appengine_config.py`.
-Flask==0.10.1
+Flask-Cors==2.1.2
gunicorn==19.3.0
Jinja2==2.8
Werkzeug==0.10.4
From 8aa1de25a4f96528796885958fd5557d5cc0e81f Mon Sep 17 00:00:00 2001
From: Brian Thomas
Date: Tue, 5 Jan 2016 14:51:43 -0500
Subject: [PATCH 003/146] Remove custom CORS solution
---
apod/service.py | 12 +-----------
1 file changed, 1 insertion(+), 11 deletions(-)
diff --git a/apod/service.py b/apod/service.py
index 7347476..3886457 100644
--- a/apod/service.py
+++ b/apod/service.py
@@ -190,21 +190,11 @@ def home():
methodname=APOD_METHOD_NAME, \
usage=_usage(joinstr='", "', prestr='"')+'"')
-@app.route('/'+SERVICE_VERSION+'/'+APOD_METHOD_NAME+'/', methods=['GET','OPTIONS'])
+@app.route('/'+SERVICE_VERSION+'/'+APOD_METHOD_NAME+'/', methods=['GET'])
def apod():
try:
- # trap OPTIONS method to handle the x-site issue
- if request.method == "OPTIONS":
- response = Response("", status=200, mimetype='application/json')
- response.headers['Access-Control-Allow-Origin'] = '*'
- response.headers['Access-Control-Allow-Methods'] = 'POST, OPTIONS'
- response.headers['Access-Control-Max-Age'] = 1000
- # note that '*' is not valid for Access-Control-Allow-Headers
- response.headers['Access-Control-Allow-Headers'] = 'origin, x-csrftoken, content-type, accept'
- return response
-
# application/json GET method
args = request.args
From dd6d585a64260edf8d8bad169f762f1d172ce4ea Mon Sep 17 00:00:00 2001
From: Brian Thomas
Date: Thu, 7 Jan 2016 14:46:38 -0500
Subject: [PATCH 004/146] Add back in hd parameter and hdurl return param
The hd parameter is now simply ignored, we always return
hdurl regardless.
---
apod/service.py | 21 +++++++++++++++++----
1 file changed, 17 insertions(+), 4 deletions(-)
diff --git a/apod/service.py b/apod/service.py
index 3886457..f28d4bd 100644
--- a/apod/service.py
+++ b/apod/service.py
@@ -10,19 +10,22 @@
from bs4 import BeautifulSoup
from datetime import datetime
-from flask import request, jsonify, render_template, Response, Flask
+from flask import request, jsonify, render_template, Flask
from flask.ext.cors import CORS
import json
import requests
+import logging
app = Flask(__name__)
CORS(app)
+LOG = logging.getLogger(__name__)
+
# this should reflect both this service and the backing
# assorted libraries
SERVICE_VERSION='v1'
APOD_METHOD_NAME='apod'
-ALLOWED_APOD_FIELDS = ['concept_tags', 'date']
+ALLOWED_APOD_FIELDS = ['concept_tags', 'date', 'hd']
ALCHEMY_API_KEY = None
# location of backing APOD service
@@ -63,7 +66,16 @@ def _apod_characteristics(date):
url = '%sap%s.html' % (BASE, date_str)
soup = BeautifulSoup(requests.get(url).text, "html.parser")
suffix = soup.img['src']
- return _explanation(soup), _title(soup), _copyright(soup), BASE + suffix
+ hd_suffix = suffix
+
+ for link in soup.find_all('a', href=True):
+ print ("link:"+str(link))
+ if link['href'] and link['href'].startswith("image"):
+ print (" href:"+str(link['href']))
+ hd_suffix = link['href']
+ break
+
+ return _explanation(soup), _title(soup), _copyright(soup), BASE + suffix, BASE + hd_suffix
except Exception as ex:
print ("EXCEPTION: "+str(ex))
@@ -76,10 +88,11 @@ def _apod_handler(date, use_concept_tags=False):
try:
d = {}
d['date'] = date
- explanation, title, copyright, url = _apod_characteristics(date)
+ explanation, title, copyright, url, hdurl = _apod_characteristics(date)
d['explanation'] = explanation
d['title'] = title
d['url'] = url
+ d['hdurl'] = hdurl
if copyright:
d['copyright'] = copyright
if use_concept_tags:
From 093a64f357456ac703967ae9ea9b0290a7c86770 Mon Sep 17 00:00:00 2001
From: Brian Thomas
Date: Thu, 7 Jan 2016 15:13:10 -0500
Subject: [PATCH 005/146] Fix missing media_type and handle video content
---
apod/service.py | 42 +++++++++++++++++++++++++++++-------------
1 file changed, 29 insertions(+), 13 deletions(-)
diff --git a/apod/service.py b/apod/service.py
index f28d4bd..8754b3d 100644
--- a/apod/service.py
+++ b/apod/service.py
@@ -50,6 +50,8 @@ def _abort(code, msg, usage=True):
def _apod_characteristics(date):
"""Accepts a date in '%Y-%m-%d' format. Returns the URL of the APOD image
of that day, noting that """
+
+ print("apod chars called")
today = datetime.today()
begin = datetime (1995, 6, 16) # first APOD image date
dt = datetime.strptime(date, '%Y-%m-%d')
@@ -62,20 +64,30 @@ def _apod_characteristics(date):
else:
try:
+ media_type = 'image'
date_str = dt.strftime('%y%m%d')
- url = '%sap%s.html' % (BASE, date_str)
- soup = BeautifulSoup(requests.get(url).text, "html.parser")
- suffix = soup.img['src']
- hd_suffix = suffix
-
- for link in soup.find_all('a', href=True):
- print ("link:"+str(link))
- if link['href'] and link['href'].startswith("image"):
- print (" href:"+str(link['href']))
- hd_suffix = link['href']
- break
+ apod_url = '%sap%s.html' % (BASE, date_str)
+ print ("OPENING URL:"+apod_url)
+ soup = BeautifulSoup(requests.get(apod_url).text, "html.parser")
+ print ("getting the data url")
+ data = None
+ hd_data = None
+ if soup.img:
+ # it is an image, so get both the low- and high-resolution data
+ data = BASE + soup.img['src']
+ hd_data = data
+
+ print ("getting the link for hd_data")
+ for link in soup.find_all('a', href=True):
+ if link['href'] and link['href'].startswith("image"):
+ hd_data = BASE + link['href']
+ break
+ else:
+ # its a video
+ media_type = 'video'
+ data = soup.iframe['src']
- return _explanation(soup), _title(soup), _copyright(soup), BASE + suffix, BASE + hd_suffix
+ return _explanation(soup), _title(soup), _copyright(soup), data, hd_data, media_type
except Exception as ex:
print ("EXCEPTION: "+str(ex))
@@ -88,11 +100,12 @@ def _apod_handler(date, use_concept_tags=False):
try:
d = {}
d['date'] = date
- explanation, title, copyright, url, hdurl = _apod_characteristics(date)
+ explanation, title, copyright, url, hdurl, media_type = _apod_characteristics(date)
d['explanation'] = explanation
d['title'] = title
d['url'] = url
d['hdurl'] = hdurl
+ d['media_type'] = media_type
if copyright:
d['copyright'] = copyright
if use_concept_tags:
@@ -132,6 +145,7 @@ def _title(soup):
"""Accepts a BeautifulSoup object for the APOD HTML page and returns the
APOD image title. Highly idiosyncratic with adaptations for different
HTML structures that appear over time."""
+ print ("getting the title")
try:
# Handler for later APOD entries
center_selection = soup.find_all('center')[1]
@@ -148,6 +162,7 @@ def _copyright(soup):
"""Accepts a BeautifulSoup object for the APOD HTML page and returns the
APOD image copyright. Highly idiosyncratic with adaptations for different
HTML structures that appear over time."""
+ print ("getting the copyright")
try:
# Handler for later APOD entries
center_selection = soup.find_all('center')[1]
@@ -169,6 +184,7 @@ def _explanation(soup):
"""Accepts a BeautifulSoup object for the APOD HTML page and returns the
APOD image explanation. Highly idiosyncratic."""
# Handler for later APOD entries
+ print ("getting the explanation")
s = soup.find_all('p')[2].text
s = s.replace('\n', ' ')
s = s.replace(' ', ' ')
From edd61c4c456038d5b64bbcd374ac972d9c6eb35c Mon Sep 17 00:00:00 2001
From: Brian Thomas
Date: Thu, 7 Jan 2016 15:21:38 -0500
Subject: [PATCH 006/146] Update README for added hd, media_type and hdurl
params
---
README.md | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/README.md b/README.md
index 18bc11e..69bf344 100644
--- a/README.md
+++ b/README.md
@@ -12,6 +12,7 @@ as parameters to a http GET request. A JSON dictionary is returned nominally.
- `date` A string in YYYY-MM-DD format indicating the date of the APOD image (example: 2014-11-03). Defaults to today's date. Must be after 1995-06-16, the first day an APOD picture was posted. There are no images for tomorrow available through this API.
- `concept_tags` A boolean indicating whether concept tags should be returned with the rest of the response. The concept tags are not necessarily included in the explanation, but rather derived from common search tags that are associated with the description text. (Better than just pure text search.) Defaults to False.
+- `hd` A boolean parameter indicating whether or not high-resolution images should be returned. This is present for legacy purposes, it is always ignored by the service and high-resolution urls are returned regardless.
**Returned fields**
@@ -19,7 +20,9 @@ as parameters to a http GET request. A JSON dictionary is returned nominally.
- `concept_tags` A boolean reflection of the supplied option. Included in response because of default values.
- `title` The title of the image.
- `date` Date of image. Included in response because of default values.
-- `url` The URL of the APOD image of the day.
+- `url` The URL of the APOD image or video of the day.
+- `hdurl` The URL for any high-resolution image for that day. Always returned but the value will be 'null' on dates which have video.
+- `media_type` The type of media (data) returned. May either be 'image' or 'video' depending on content.
- `explanation` The supplied text explanation of the image.
- `concepts` The most relevant concepts within the text explanation. Only supplied if `concept_tags` is set to True.
From 8411fc93982c7dbb2e864ebfa42c1c4552e273c8 Mon Sep 17 00:00:00 2001
From: Brian Thomas
Date: Thu, 7 Jan 2016 15:28:47 -0500
Subject: [PATCH 007/146] Omit returning hdurl when it does not exist per
request on GitHub
---
README.md | 2 +-
apod/service.py | 3 ++-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/README.md b/README.md
index 69bf344..f84d1cb 100644
--- a/README.md
+++ b/README.md
@@ -21,7 +21,7 @@ as parameters to a http GET request. A JSON dictionary is returned nominally.
- `title` The title of the image.
- `date` Date of image. Included in response because of default values.
- `url` The URL of the APOD image or video of the day.
-- `hdurl` The URL for any high-resolution image for that day. Always returned but the value will be 'null' on dates which have video.
+- `hdurl` The URL for any high-resolution image for that day. Returned regardless of 'hd' param setting but will be ommited in the response IF it does not exist originally at APOD.
- `media_type` The type of media (data) returned. May either be 'image' or 'video' depending on content.
- `explanation` The supplied text explanation of the image.
- `concepts` The most relevant concepts within the text explanation. Only supplied if `concept_tags` is set to True.
diff --git a/apod/service.py b/apod/service.py
index 8754b3d..1102129 100644
--- a/apod/service.py
+++ b/apod/service.py
@@ -104,7 +104,8 @@ def _apod_handler(date, use_concept_tags=False):
d['explanation'] = explanation
d['title'] = title
d['url'] = url
- d['hdurl'] = hdurl
+ if hdurl:
+ d['hdurl'] = hdurl
d['media_type'] = media_type
if copyright:
d['copyright'] = copyright
From 823f02eea9396fa96f4e6c43c94e9f64dba71dc8 Mon Sep 17 00:00:00 2001
From: brianthomas
Date: Fri, 15 Jan 2016 14:04:03 -0500
Subject: [PATCH 008/146] remove print stmts in favor of LOG
---
apod/service.py | 26 ++++++++++++--------------
1 file changed, 12 insertions(+), 14 deletions(-)
diff --git a/apod/service.py b/apod/service.py
index 1102129..0d94207 100644
--- a/apod/service.py
+++ b/apod/service.py
@@ -35,7 +35,7 @@
with open('alchemy_api.key', 'r') as f:
ALCHEMY_API_KEY = f.read()
except:
- print ("WARNING: NO alchemy_api.key found, concept_tagging is NOT supported")
+ LOG.info ("WARNING: NO alchemy_api.key found, concept_tagging is NOT supported")
def _abort(code, msg, usage=True):
@@ -44,14 +44,14 @@ def _abort(code, msg, usage=True):
response = jsonify(service_version=SERVICE_VERSION, msg=msg)
response.status_code = code
- print (str(response))
+ LOG.debug(str(response))
return response
def _apod_characteristics(date):
"""Accepts a date in '%Y-%m-%d' format. Returns the URL of the APOD image
of that day, noting that """
- print("apod chars called")
+ LOG.debug("apod chars called")
today = datetime.today()
begin = datetime (1995, 6, 16) # first APOD image date
dt = datetime.strptime(date, '%Y-%m-%d')
@@ -67,9 +67,9 @@ def _apod_characteristics(date):
media_type = 'image'
date_str = dt.strftime('%y%m%d')
apod_url = '%sap%s.html' % (BASE, date_str)
- print ("OPENING URL:"+apod_url)
+ LOG.debug("OPENING URL:"+apod_url)
soup = BeautifulSoup(requests.get(apod_url).text, "html.parser")
- print ("getting the data url")
+ LOG.debug("getting the data url")
data = None
hd_data = None
if soup.img:
@@ -77,7 +77,7 @@ def _apod_characteristics(date):
data = BASE + soup.img['src']
hd_data = data
- print ("getting the link for hd_data")
+ LOG.debug("getting the link for hd_data")
for link in soup.find_all('a', href=True):
if link['href'] and link['href'].startswith("image"):
hd_data = BASE + link['href']
@@ -90,7 +90,7 @@ def _apod_characteristics(date):
return _explanation(soup), _title(soup), _copyright(soup), data, hd_data, media_type
except Exception as ex:
- print ("EXCEPTION: "+str(ex))
+ LOG.error("Caught exception type:"+str(type(ex))+" msg:"+str(ex))
# this most probably should return code 500 here
raise ValueError('No APOD imagery for the given date.')
@@ -132,13 +132,12 @@ def _concepts(text, apikey):
try:
- print ("Getting response")
+ LOG.debug("Getting response")
response = json.loads(request.get(cbase, fields=params))
clist = [concept['text'] for concept in response['concepts']]
return {k: v for k, v in zip(range(len(clist)), clist)}
except Exception as ex:
- print (str(ex))
raise ValueError(ex)
@@ -146,7 +145,7 @@ def _title(soup):
"""Accepts a BeautifulSoup object for the APOD HTML page and returns the
APOD image title. Highly idiosyncratic with adaptations for different
HTML structures that appear over time."""
- print ("getting the title")
+ LOG.debug("getting the title")
try:
# Handler for later APOD entries
center_selection = soup.find_all('center')[1]
@@ -163,7 +162,7 @@ def _copyright(soup):
"""Accepts a BeautifulSoup object for the APOD HTML page and returns the
APOD image copyright. Highly idiosyncratic with adaptations for different
HTML structures that appear over time."""
- print ("getting the copyright")
+ LOG.debug("getting the copyright")
try:
# Handler for later APOD entries
center_selection = soup.find_all('center')[1]
@@ -185,7 +184,7 @@ def _explanation(soup):
"""Accepts a BeautifulSoup object for the APOD HTML page and returns the
APOD image explanation. Highly idiosyncratic."""
# Handler for later APOD entries
- print ("getting the explanation")
+ LOG.debug("getting the explanation")
s = soup.find_all('p')[2].text
s = s.replace('\n', ' ')
s = s.replace(' ', ' ')
@@ -244,11 +243,10 @@ def apod():
except Exception as ex:
etype = type(ex)
- #print (str(etype)+"\n "+str(ex))
if etype == ValueError or "BadRequest" in str(etype):
return _abort(400, str(ex)+".")
else:
- print ("Service Exception. Msg: "+str(type(ex)))
+ LOG.error("Service Exception. Msg: "+str(type(ex)))
return _abort(500, "Internal Service Error", usage=False)
@app.errorhandler(404)
From 5367c266ac062937ff6d8700df738208c1799bcc Mon Sep 17 00:00:00 2001
From: brianthomas
Date: Fri, 15 Jan 2016 14:38:50 -0500
Subject: [PATCH 009/146] Try to fix issue where the Heroku service uses
a different timezone from the underlying APOD service, the
fallback is to try one day earlier
---
apod/service.py | 141 ++++++++++++++++++++++++++++++------------------
1 file changed, 90 insertions(+), 51 deletions(-)
diff --git a/apod/service.py b/apod/service.py
index 0d94207..f668b1b 100644
--- a/apod/service.py
+++ b/apod/service.py
@@ -9,7 +9,7 @@
'''
from bs4 import BeautifulSoup
-from datetime import datetime
+from datetime import datetime, timedelta
from flask import request, jsonify, render_template, Flask
from flask.ext.cors import CORS
import json
@@ -42,65 +42,71 @@ def _abort(code, msg, usage=True):
if (usage):
msg += " "+_usage()+"'"
- response = jsonify(service_version=SERVICE_VERSION, msg=msg)
+ response = jsonify(service_version=SERVICE_VERSION, msg=msg, code=code)
response.status_code = code
LOG.debug(str(response))
+
return response
-def _apod_characteristics(date):
+def _get_apod_chars(dt):
+
+ media_type = 'image'
+ date_str = dt.strftime('%y%m%d')
+ apod_url = '%sap%s.html' % (BASE, date_str)
+ LOG.debug("OPENING URL:"+apod_url)
+ soup = BeautifulSoup(requests.get(apod_url).text, "html.parser")
+ LOG.debug("getting the data url")
+ data = None
+ hd_data = None
+ if soup.img:
+ # it is an image, so get both the low- and high-resolution data
+ data = BASE + soup.img['src']
+ hd_data = data
+
+ LOG.debug("getting the link for hd_data")
+ for link in soup.find_all('a', href=True):
+ if link['href'] and link['href'].startswith("image"):
+ hd_data = BASE + link['href']
+ break
+ else:
+ # its a video
+ media_type = 'video'
+ data = soup.iframe['src']
+
+ return _explanation(soup), _title(soup), _copyright(soup), data, hd_data, media_type
+
+
+def _apod_characteristics(dt, use_default_today_date=False):
"""Accepts a date in '%Y-%m-%d' format. Returns the URL of the APOD image
of that day, noting that """
LOG.debug("apod chars called")
- today = datetime.today()
- begin = datetime (1995, 6, 16) # first APOD image date
- dt = datetime.strptime(date, '%Y-%m-%d')
- if (dt > today) or (dt < begin):
- today_str = today.strftime('%b %d, %Y')
- begin_str = begin.strftime('%b %d, %Y')
- raise ValueError(
- 'Date must be between %s and %s.' % (begin_str, today_str)
- )
- else:
- try:
-
- media_type = 'image'
- date_str = dt.strftime('%y%m%d')
- apod_url = '%sap%s.html' % (BASE, date_str)
- LOG.debug("OPENING URL:"+apod_url)
- soup = BeautifulSoup(requests.get(apod_url).text, "html.parser")
- LOG.debug("getting the data url")
- data = None
- hd_data = None
- if soup.img:
- # it is an image, so get both the low- and high-resolution data
- data = BASE + soup.img['src']
- hd_data = data
-
- LOG.debug("getting the link for hd_data")
- for link in soup.find_all('a', href=True):
- if link['href'] and link['href'].startswith("image"):
- hd_data = BASE + link['href']
- break
- else:
- # its a video
- media_type = 'video'
- data = soup.iframe['src']
-
- return _explanation(soup), _title(soup), _copyright(soup), data, hd_data, media_type
+
+ try:
- except Exception as ex:
- LOG.error("Caught exception type:"+str(type(ex))+" msg:"+str(ex))
- # this most probably should return code 500 here
- raise ValueError('No APOD imagery for the given date.')
-
-def _apod_handler(date, use_concept_tags=False):
+ return _get_apod_chars(dt)
+
+ except Exception as ex:
+
+ # handle edge case where the service local time
+ # miss-matches with 'todays date' of the underlying APOD
+ # service (can happen because they are deployed in different
+ # timezones). Use the fallback of prior day's date
+
+ if use_default_today_date:
+ # try to get the day before
+ dt = dt - timedelta(days=1)
+ return _get_apod_chars(dt)
+ else:
+ # pass exception up the call stack
+ raise Exception(ex)
+
+def _apod_handler(dt, use_concept_tags=False, use_default_today_date=False):
"""Accepts a parameter dictionary. Returns the response object to be
served through the API."""
try:
d = {}
- d['date'] = date
- explanation, title, copyright, url, hdurl, media_type = _apod_characteristics(date)
+ explanation, title, copyright, url, hdurl, media_type = _apod_characteristics(dt, use_default_today_date)
d['explanation'] = explanation
d['title'] = title
d['url'] = url
@@ -115,9 +121,12 @@ def _apod_handler(date, use_concept_tags=False):
else:
d['concepts'] = _concepts(explanation, ALCHEMY_API_KEY)
return d
+
except Exception as e:
- m = 'Your request could not be processed.'
- return dict(message=m, error=str(e))
+
+ LOG.error("Internal Service Error :"+str(type(e))+" msg:"+str(e))
+ # return code 500 here
+ return _abort(500, "Internal Service Error", usage=False)
def _concepts(text, apikey):
"""Returns the concepts associated with the text, interleaved with integer
@@ -209,6 +218,20 @@ def _validate (data):
return False
return True
+def _validate_date (dt):
+
+ today = datetime.today()
+ begin = datetime (1995, 6, 16) # first APOD image date
+
+ # validate input
+ if (dt > today) or (dt < begin):
+
+ today_str = today.strftime('%b %d, %Y')
+ begin_str = begin.strftime('%b %d, %Y')
+
+ raise ValueError('Date must be between %s and %s.' % (begin_str, today_str))
+
+
# Endpoints
#
@@ -230,16 +253,32 @@ def apod():
if not _validate(args):
return _abort (400, "Bad Request incorrect field passed.")
- date = args.get('date', datetime.strftime(datetime.today(), '%Y-%m-%d'))
+ # get the date param
+ use_default_today_date = False
+ date = args.get('date')
+ if not date:
+ # fall back to using today's date IF they didn't specify a date
+ date = datetime.strftime(datetime.today(), '%Y-%m-%d')
+ use_default_today_date = True
+
+ # grab the concept_tags param
use_concept_tags = args.get('concept_tags', False)
+ # validate input date
+ dt = datetime.strptime(date, '%Y-%m-%d')
+ _validate_date(dt)
+
# get data
- data = _apod_handler(date, use_concept_tags)
+ data = _apod_handler(dt, use_concept_tags, use_default_today_date)
+ data['date'] = date
data['service_version'] = SERVICE_VERSION
# return info as JSON
return jsonify(data)
+ except ValueError as ve:
+ return _abort(400, str(ve), False)
+
except Exception as ex:
etype = type(ex)
From e49d8c727248a5fb66c4390dee3e5e70ec13e21b Mon Sep 17 00:00:00 2001
From: Brian Thomas
Date: Thu, 23 Mar 2017 14:39:28 -0400
Subject: [PATCH 010/146] Ignore env
---
.gitignore | 1 +
1 file changed, 1 insertion(+)
diff --git a/.gitignore b/.gitignore
index 5d153f2..add9a2b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
*.pyc
.project
.pydevproject
+env/
alchemy_api.key
# don't include third-party dependencies.
lib/
From 643076565c01d85dd0ecf8c7c979d2c71a815030 Mon Sep 17 00:00:00 2001
From: Brian Thomas
Date: Thu, 23 Mar 2017 14:40:18 -0400
Subject: [PATCH 011/146] Try to fix parsing of the copyright
---
apod/service.py | 40 +++++++++++++++++++++++++++-------------
1 file changed, 27 insertions(+), 13 deletions(-)
diff --git a/apod/service.py b/apod/service.py
index f668b1b..8ee2004 100644
--- a/apod/service.py
+++ b/apod/service.py
@@ -20,6 +20,8 @@
CORS(app)
LOG = logging.getLogger(__name__)
+logging.basicConfig(level=logging.WARN)
+#LOG.setLevel(logging.DEBUG)
# this should reflect both this service and the backing
# assorted libraries
@@ -107,6 +109,7 @@ def _apod_handler(dt, use_concept_tags=False, use_default_today_date=False):
try:
d = {}
explanation, title, copyright, url, hdurl, media_type = _apod_characteristics(dt, use_default_today_date)
+ LOG.debug("managed to get apod characteristics")
d['explanation'] = explanation
d['title'] = title
d['url'] = url
@@ -174,21 +177,32 @@ def _copyright(soup):
LOG.debug("getting the copyright")
try:
# Handler for later APOD entries
- center_selection = soup.find_all('center')[1]
- bold_selection = center_selection.find_all('b')[1]
- if "Copyright" in bold_selection.text:
- # pull the copyright from the link text
- link_selection = center_selection.find_all('a')[0]
- if "Copyright" in link_selection.text:
- # hmm. older style, try to grab from 2nd link
- link_selection = center_selection.find_all('a')[1]
- return link_selection.text.strip(' ')
- else:
- # NO stated copyright, so we return None
- return None
- except Exception:
+
+ # There's no uniform handling of copyright (sigh). Well, we just have to
+ # try every stinking text block we find...
+
+ for element in soup.findAll('b', text=True):
+ #LOG.debug("TEXT: "+element.text)
+ # search text for explicit match
+ if "Copyright" in element.text:
+ LOG.debug("Found Copyright text:"+str(element.text))
+ LOG.debug(" element:"+str(element))
+ # pull the copyright from the link text
+ link_selection = element.parent.find_all('a')[0]
+ if "Copyright" in link_selection.text:
+ # hmm. older style, try to grab from 2nd link
+ LOG.debug("trying olderstyle copyright grab")
+ link_selection = element.parent.find_all('a')[1]
+ # return
+ return link_selection.text.strip(' ')
+
+ except Exception as ex:
+ LOG.error(str(ex))
raise ValueError('Unsupported schema for given date.')
+ # NO stated copyright, so we return None
+ return None
+
def _explanation(soup):
"""Accepts a BeautifulSoup object for the APOD HTML page and returns the
APOD image explanation. Highly idiosyncratic."""
From cd807aad88f4acf4b9457e604c1fc6ff0edbb2b5 Mon Sep 17 00:00:00 2001
From: brianthomas
Date: Fri, 24 Mar 2017 16:07:46 -0400
Subject: [PATCH 012/146] ignore some test files
---
.gitignore | 1 +
1 file changed, 1 insertion(+)
diff --git a/.gitignore b/.gitignore
index add9a2b..33c76c7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
*.pyc
.project
.pydevproject
+.coverage/
env/
alchemy_api.key
# don't include third-party dependencies.
From 5c0715f198a991a177192806caf6b151de9b5b33 Mon Sep 17 00:00:00 2001
From: brianthomas
Date: Fri, 24 Mar 2017 16:08:38 -0400
Subject: [PATCH 013/146] update requirements, bs4 version and coverage/tests
reqs
---
requirements.txt | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/requirements.txt b/requirements.txt
index 5588427..5d9e82b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,5 +8,8 @@ Flask-Cors==2.1.2
gunicorn==19.3.0
Jinja2==2.8
Werkzeug==0.10.4
-beautifulsoup4==4.4.1
+beautifulsoup4==4.5.3
requests==2.8.1
+coverage==4.1
+nose==1.3.7
+setupext-janitor==1.0.0
From b79879e2008a34b97f64536cf203137f2c8246f1 Mon Sep 17 00:00:00 2001
From: brianthomas
Date: Fri, 24 Mar 2017 16:09:24 -0400
Subject: [PATCH 014/146] add setup.py
---
setup.py | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 69 insertions(+)
create mode 100644 setup.py
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..eff23bd
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,69 @@
+
+from os.path import dirname, join
+from setuptools import setup, find_packages, Command
+
+with open('requirements.txt') as f:
+ reqs = f.read().splitlines()
+
+'''
+# Implement setupext.janitor which allows for more flexible
+# and powerful cleaning. Commands include:
+
+setup.py clean --dist
+ Removes directories that the various dist commands produce.
+setup.py clean --egg
+ Removes .egg and .egg-info directories.
+setup.py clean --environment
+ Removes the currently active virtual environment as indicated by the $VIRTUAL_ENV environment variable. The name of the directory can also be specified using the --virtualenv-dir command line option.
+setup.py clean --pycache
+ Recursively removes directories named __pycache__.
+setup.py clean --all
+ Remove all of by-products. This is the same as using --dist --egg --environment --pycache.
+'''
+
+try:
+ from setupext import janitor
+ CleanCommand = janitor.CleanCommand
+except ImportError:
+ CleanCommand = None
+
+cmd_classes = {}
+if CleanCommand is not None:
+ cmd_classes['clean'] = CleanCommand
+
+with open(join(dirname(__file__), 'README.md'), 'rb') as f:
+ long_description = f.read().decode('ascii').strip()
+
+import os
+scripts = [os.path.join("bin",file) for file in os.listdir("bin")]
+
+import apod
+version=apod.version
+
+setup (
+
+ name='apod-api',
+ description='Python microservice for APOD site',
+ url='https://www.github.com/nasa/apod-api',
+ version=version,
+
+ keywords = 'apod api nasa python',
+ long_description=long_description,
+
+ scripts=scripts,
+
+ maintainer='Brian Thomas',
+ maintainer_email='brian.a.thomas@nasa.gov',
+
+ packages=find_packages(exclude=('tests', 'tests.*')),
+ license='Apache2 License',
+
+ include_package_data=True,
+
+ setup_requires=['setupext-janitor'],
+ cmdclass = cmd_classes,
+
+ install_requires=reqs,
+
+)
+
From c46d201fe308a61f7a400afe9be319434593fb73 Mon Sep 17 00:00:00 2001
From: brianthomas
Date: Fri, 24 Mar 2017 16:10:10 -0400
Subject: [PATCH 015/146] ignore .coverage file
---
.gitignore | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitignore b/.gitignore
index 33c76c7..9c7a6fa 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,7 @@
*.pyc
.project
.pydevproject
-.coverage/
+.coverage
env/
alchemy_api.key
# don't include third-party dependencies.
From 3663727e2c880d3a9a4870ce1e140432ec0de479 Mon Sep 17 00:00:00 2001
From: brianthomas
Date: Fri, 24 Mar 2017 16:10:31 -0400
Subject: [PATCH 016/146] fix up directory init/path and add unit tests with
coverage
---
apod/__init__.py | 0
apod/service.py | 208 ++++++-------------------------------
apod/tests/apod_test.py | 15 ---
apod/utility.py | 165 +++++++++++++++++++++++++++++
run_coverage.sh | 3 +
setup.cfg | 2 +-
tests/__init__.py | 0
tests/apod/__init__.py | 0
tests/apod/test_utility.py | 33 ++++++
9 files changed, 235 insertions(+), 191 deletions(-)
create mode 100644 apod/__init__.py
delete mode 100644 apod/tests/apod_test.py
create mode 100644 apod/utility.py
create mode 100644 run_coverage.sh
create mode 100644 tests/__init__.py
create mode 100644 tests/apod/__init__.py
create mode 100644 tests/apod/test_utility.py
diff --git a/apod/__init__.py b/apod/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/apod/service.py b/apod/service.py
index 8ee2004..7aba614 100644
--- a/apod/service.py
+++ b/apod/service.py
@@ -8,19 +8,17 @@
@author=bathomas @email=brian.a.thomas@nasa.gov
'''
-from bs4 import BeautifulSoup
-from datetime import datetime, timedelta
+from datetime import datetime
from flask import request, jsonify, render_template, Flask
-from flask.ext.cors import CORS
-import json
-import requests
+from flask_cors import CORS, cross_origin
+from utility import parse_apod, get_concepts
import logging
app = Flask(__name__)
CORS(app)
LOG = logging.getLogger(__name__)
-logging.basicConfig(level=logging.WARN)
+logging.basicConfig(level=logging.DEBUG)
#LOG.setLevel(logging.DEBUG)
# this should reflect both this service and the backing
@@ -30,8 +28,6 @@
ALLOWED_APOD_FIELDS = ['concept_tags', 'date', 'hd']
ALCHEMY_API_KEY = None
-# location of backing APOD service
-BASE = 'http://apod.nasa.gov/apod/'
try:
with open('alchemy_api.key', 'r') as f:
@@ -50,79 +46,54 @@ def _abort(code, msg, usage=True):
return response
-def _get_apod_chars(dt):
-
- media_type = 'image'
- date_str = dt.strftime('%y%m%d')
- apod_url = '%sap%s.html' % (BASE, date_str)
- LOG.debug("OPENING URL:"+apod_url)
- soup = BeautifulSoup(requests.get(apod_url).text, "html.parser")
- LOG.debug("getting the data url")
- data = None
- hd_data = None
- if soup.img:
- # it is an image, so get both the low- and high-resolution data
- data = BASE + soup.img['src']
- hd_data = data
-
- LOG.debug("getting the link for hd_data")
- for link in soup.find_all('a', href=True):
- if link['href'] and link['href'].startswith("image"):
- hd_data = BASE + link['href']
- break
- else:
- # its a video
- media_type = 'video'
- data = soup.iframe['src']
-
- return _explanation(soup), _title(soup), _copyright(soup), data, hd_data, media_type
-
-
-def _apod_characteristics(dt, use_default_today_date=False):
- """Accepts a date in '%Y-%m-%d' format. Returns the URL of the APOD image
- of that day, noting that """
+def _usage(joinstr="', '", prestr="'"):
+ return "Allowed request fields for "+APOD_METHOD_NAME+" method are "+prestr+joinstr.join(ALLOWED_APOD_FIELDS)
- LOG.debug("apod chars called")
+def _validate (data):
+ LOG.debug("_validate(data) called")
+ for key in data:
+ if key not in ALLOWED_APOD_FIELDS:
+ return False
+ return True
+
+def _validate_date (dt):
- try:
-
- return _get_apod_chars(dt)
+ LOG.debug("_validate_date(dt) called")
+ today = datetime.today()
+ begin = datetime (1995, 6, 16) # first APOD image date
- except Exception as ex:
+ # validate input
+ if (dt > today) or (dt < begin):
- # handle edge case where the service local time
- # miss-matches with 'todays date' of the underlying APOD
- # service (can happen because they are deployed in different
- # timezones). Use the fallback of prior day's date
+ today_str = today.strftime('%b %d, %Y')
+ begin_str = begin.strftime('%b %d, %Y')
+
+ raise ValueError('Date must be between %s and %s.' % (begin_str, today_str))
- if use_default_today_date:
- # try to get the day before
- dt = dt - timedelta(days=1)
- return _get_apod_chars(dt)
- else:
- # pass exception up the call stack
- raise Exception(ex)
-
def _apod_handler(dt, use_concept_tags=False, use_default_today_date=False):
"""Accepts a parameter dictionary. Returns the response object to be
served through the API."""
try:
d = {}
- explanation, title, copyright, url, hdurl, media_type = _apod_characteristics(dt, use_default_today_date)
+ explanation, title, copyrght, url, hdurl, media_type = parse_apod(dt, use_default_today_date)
LOG.debug("managed to get apod characteristics")
+
d['explanation'] = explanation
d['title'] = title
d['url'] = url
if hdurl:
d['hdurl'] = hdurl
d['media_type'] = media_type
- if copyright:
- d['copyright'] = copyright
+
+ if copyrght:
+ d['copyright'] = copyrght
+
if use_concept_tags:
if ALCHEMY_API_KEY == None:
d['concepts'] = "concept_tags functionality turned off in current service"
else:
- d['concepts'] = _concepts(explanation, ALCHEMY_API_KEY)
+ d['concepts'] = get_concepts(request, explanation, ALCHEMY_API_KEY)
+
return d
except Exception as e:
@@ -131,121 +102,7 @@ def _apod_handler(dt, use_concept_tags=False, use_default_today_date=False):
# return code 500 here
return _abort(500, "Internal Service Error", usage=False)
-def _concepts(text, apikey):
- """Returns the concepts associated with the text, interleaved with integer
- keys indicating the index."""
- cbase = 'http://access.alchemyapi.com/calls/text/TextGetRankedConcepts'
-
- params = dict(
- outputMode='json',
- apikey=apikey,
- text=text
- )
-
- try:
-
- LOG.debug("Getting response")
- response = json.loads(request.get(cbase, fields=params))
- clist = [concept['text'] for concept in response['concepts']]
- return {k: v for k, v in zip(range(len(clist)), clist)}
-
- except Exception as ex:
- raise ValueError(ex)
-
-
-def _title(soup):
- """Accepts a BeautifulSoup object for the APOD HTML page and returns the
- APOD image title. Highly idiosyncratic with adaptations for different
- HTML structures that appear over time."""
- LOG.debug("getting the title")
- try:
- # Handler for later APOD entries
- center_selection = soup.find_all('center')[1]
- bold_selection = center_selection.find_all('b')[0]
- return bold_selection.text.strip(' ')
- except Exception:
- # Handler for early APOD entries
- text = soup.title.text.split(' - ')[-1]
- return text.strip()
- else:
- raise ValueError('Unsupported schema for given date.')
-
-def _copyright(soup):
- """Accepts a BeautifulSoup object for the APOD HTML page and returns the
- APOD image copyright. Highly idiosyncratic with adaptations for different
- HTML structures that appear over time."""
- LOG.debug("getting the copyright")
- try:
- # Handler for later APOD entries
-
- # There's no uniform handling of copyright (sigh). Well, we just have to
- # try every stinking text block we find...
-
- for element in soup.findAll('b', text=True):
- #LOG.debug("TEXT: "+element.text)
- # search text for explicit match
- if "Copyright" in element.text:
- LOG.debug("Found Copyright text:"+str(element.text))
- LOG.debug(" element:"+str(element))
- # pull the copyright from the link text
- link_selection = element.parent.find_all('a')[0]
- if "Copyright" in link_selection.text:
- # hmm. older style, try to grab from 2nd link
- LOG.debug("trying olderstyle copyright grab")
- link_selection = element.parent.find_all('a')[1]
- # return
- return link_selection.text.strip(' ')
-
- except Exception as ex:
- LOG.error(str(ex))
- raise ValueError('Unsupported schema for given date.')
-
- # NO stated copyright, so we return None
- return None
-
-def _explanation(soup):
- """Accepts a BeautifulSoup object for the APOD HTML page and returns the
- APOD image explanation. Highly idiosyncratic."""
- # Handler for later APOD entries
- LOG.debug("getting the explanation")
- s = soup.find_all('p')[2].text
- s = s.replace('\n', ' ')
- s = s.replace(' ', ' ')
- s = s.strip(' ').strip('Explanation: ')
- s = s.split(' Tomorrow\'s picture')[0]
- s = s.split('digg_url')[0]
- s = s.strip(' ')
- if s == '':
- # Handler for earlier APOD entries
- texts = [x.strip() for x in soup.text.split('\n')]
- begin_idx = texts.index('Explanation:') + 1
- idx = texts[begin_idx:].index('')
- s = (' ').join(texts[begin_idx:begin_idx + idx])
- return s
-
-def _usage(joinstr="', '", prestr="'"):
- return "Allowed request fields for "+APOD_METHOD_NAME+" method are "+prestr+joinstr.join(ALLOWED_APOD_FIELDS)
-
-def _validate (data):
- for key in data:
- if key not in ALLOWED_APOD_FIELDS:
- return False
- return True
-
-def _validate_date (dt):
-
- today = datetime.today()
- begin = datetime (1995, 6, 16) # first APOD image date
-
- # validate input
- if (dt > today) or (dt < begin):
-
- today_str = today.strftime('%b %d, %Y')
- begin_str = begin.strftime('%b %d, %Y')
-
- raise ValueError('Date must be between %s and %s.' % (begin_str, today_str))
-
-
+#
# Endpoints
#
@@ -259,6 +116,7 @@ def home():
@app.route('/'+SERVICE_VERSION+'/'+APOD_METHOD_NAME+'/', methods=['GET'])
def apod():
+ LOG.info("apod path called")
try:
# application/json GET method
diff --git a/apod/tests/apod_test.py b/apod/tests/apod_test.py
deleted file mode 100644
index be9dbd8..0000000
--- a/apod/tests/apod_test.py
+++ /dev/null
@@ -1,15 +0,0 @@
-import unittest
-import apod
-
-
-class TestApod(unittest.TestCase):
- """Test the extraction of APOD characteristics."""
- def setUp(self):
- self.date = '2013-10-01'
-
- def test_apod_characteristics(self):
- explanation, title, copyright, url = apod._apod_characteristics(self.date)
-
- # Test returned Title
- expected_title = 'Filaments of the Vela Supernova Remnant'
- self.assertEqual(title, expected_title)
diff --git a/apod/utility.py b/apod/utility.py
new file mode 100644
index 0000000..03a70a2
--- /dev/null
+++ b/apod/utility.py
@@ -0,0 +1,165 @@
+'''
+Split off some library functions for easier testing and code management.
+
+Created on Mar 24, 2017
+
+@author=bathomas @email=brian.a.thomas@nasa.gov
+'''
+
+from bs4 import BeautifulSoup
+from datetime import timedelta
+import requests
+import logging
+import json
+
+LOG = logging.getLogger(__name__)
+logging.basicConfig(level=logging.WARN)
+#LOG.setLevel(logging.DEBUG)
+
+# location of backing APOD service
+BASE = 'http://apod.nasa.gov/apod/'
+
+def _get_apod_chars(dt):
+
+ media_type = 'image'
+ date_str = dt.strftime('%y%m%d')
+ apod_url = '%sap%s.html' % (BASE, date_str)
+ LOG.debug("OPENING URL:"+apod_url)
+ soup = BeautifulSoup(requests.get(apod_url).text, "html.parser")
+ LOG.debug("getting the data url")
+ data = None
+ hd_data = None
+ if soup.img:
+ # it is an image, so get both the low- and high-resolution data
+ data = BASE + soup.img['src']
+ hd_data = data
+
+ LOG.debug("getting the link for hd_data")
+ for link in soup.find_all('a', href=True):
+ if link['href'] and link['href'].startswith("image"):
+ hd_data = BASE + link['href']
+ break
+ else:
+ # its a video
+ media_type = 'video'
+ data = soup.iframe['src']
+
+ return _explanation(soup), _title(soup), _copyright(soup), data, hd_data, media_type
+
+def _title(soup):
+ """Accepts a BeautifulSoup object for the APOD HTML page and returns the
+ APOD image title. Highly idiosyncratic with adaptations for different
+ HTML structures that appear over time."""
+ LOG.debug("getting the title")
+ try:
+ # Handler for later APOD entries
+ center_selection = soup.find_all('center')[1]
+ bold_selection = center_selection.find_all('b')[0]
+ return bold_selection.text.strip(' ')
+ except Exception:
+ # Handler for early APOD entries
+ text = soup.title.text.split(' - ')[-1]
+ return text.strip()
+ else:
+ raise ValueError('Unsupported schema for given date.')
+
+def _copyright(soup):
+ """Accepts a BeautifulSoup object for the APOD HTML page and returns the
+ APOD image copyright. Highly idiosyncratic with adaptations for different
+ HTML structures that appear over time."""
+ LOG.debug("getting the copyright")
+ try:
+ # Handler for later APOD entries
+
+ # There's no uniform handling of copyright (sigh). Well, we just have to
+ # try every stinking text block we find...
+
+ for element in soup.findAll('b', text=True):
+ #LOG.debug("TEXT: "+element.text)
+ # search text for explicit match
+ if "Copyright" in element.text:
+ LOG.debug("Found Copyright text:"+str(element.text))
+ LOG.debug(" element:"+str(element))
+ # pull the copyright from the link text
+ link_selection = element.parent.find_all('a')[0]
+ if "Copyright" in link_selection.text:
+ # hmm. older style, try to grab from 2nd link
+ LOG.debug("trying olderstyle copyright grab")
+ link_selection = element.parent.find_all('a')[1]
+ # return
+ return link_selection.text.strip(' ')
+
+ except Exception as ex:
+ LOG.error(str(ex))
+ raise ValueError('Unsupported schema for given date.')
+
+ # NO stated copyright, so we return None
+ return None
+
+def _explanation(soup):
+ """Accepts a BeautifulSoup object for the APOD HTML page and returns the
+ APOD image explanation. Highly idiosyncratic."""
+ # Handler for later APOD entries
+ LOG.debug("getting the explanation")
+ s = soup.find_all('p')[2].text
+ s = s.replace('\n', ' ')
+ s = s.replace(' ', ' ')
+ s = s.strip(' ').strip('Explanation: ')
+ s = s.split(' Tomorrow\'s picture')[0]
+ s = s.strip(' ')
+ if s == '':
+ # Handler for earlier APOD entries
+ texts = [x.strip() for x in soup.text.split('\n')]
+ begin_idx = texts.index('Explanation:') + 1
+ idx = texts[begin_idx:].index('')
+ s = (' ').join(texts[begin_idx:begin_idx + idx])
+ return s
+
+
+def parse_apod (dt, use_default_today_date=False):
+ """Accepts a date in '%Y-%m-%d' format. Returns the URL of the APOD image
+ of that day, noting that """
+
+ LOG.debug("apod chars called date:"+str(dt))
+
+ try:
+
+ return _get_apod_chars(dt)
+
+ except Exception as ex:
+
+ # handle edge case where the service local time
+ # miss-matches with 'todays date' of the underlying APOD
+ # service (can happen because they are deployed in different
+ # timezones). Use the fallback of prior day's date
+
+ if use_default_today_date:
+ # try to get the day before
+ dt = dt - timedelta(days=1)
+ return _get_apod_chars(dt)
+ else:
+ # pass exception up the call stack
+ LOG.error(str(ex))
+ raise Exception(ex)
+
+
+def get_concepts(request, text, apikey):
+ """Returns the concepts associated with the text, interleaved with integer
+ keys indicating the index."""
+ cbase = 'http://access.alchemyapi.com/calls/text/TextGetRankedConcepts'
+
+ params = dict(
+ outputMode='json',
+ apikey=apikey,
+ text=text
+ )
+
+ try:
+
+ LOG.debug("Getting response")
+ response = json.loads(request.get(cbase, fields=params))
+ clist = [concept['text'] for concept in response['concepts']]
+ return {k: v for k, v in zip(range(len(clist)), clist)}
+
+ except Exception as ex:
+ raise ValueError(ex)
diff --git a/run_coverage.sh b/run_coverage.sh
new file mode 100644
index 0000000..abd053a
--- /dev/null
+++ b/run_coverage.sh
@@ -0,0 +1,3 @@
+# Need to sort out why this is the only way nosetests seem
+# to work right..
+nosetests -v tests/apod/*
diff --git a/setup.cfg b/setup.cfg
index 737c9c1..c8abc71 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -8,7 +8,7 @@ include=^test_*.py
# coverage
with-coverage=1
cover-branches=1
-cover-package=api
+cover-package=apod
all-modules=1
#cover-html=1
#cover-html-dir=htmlcov
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/apod/__init__.py b/tests/apod/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/apod/test_utility.py b/tests/apod/test_utility.py
new file mode 100644
index 0000000..704f0ef
--- /dev/null
+++ b/tests/apod/test_utility.py
@@ -0,0 +1,33 @@
+
+import unittest
+from apod import utility
+import logging
+
+logging.basicConfig(level=logging.DEBUG)
+
+class TestApod(unittest.TestCase):
+
+ """Test the extraction of APOD characteristics."""
+ def setUp(self):
+ from datetime import datetime
+ self.date = datetime (2013, 6, 13)
+
+ def test_apod_characteristics(self):
+ #explanation, title, url
+ values = utility.parse_apod(self.date)
+
+ # Test returned Explanation
+ expected_explan = 'You can see four planets in this serene'
+ self.assertTrue(values[0].startswith(expected_explan))
+
+ # Test returned Title
+ expected_title = 'Four Planet Sunset'
+ self.assertEqual(values[1], expected_title)
+
+ # Test returned url
+ expected_url = None
+ self.assertEqual(values[2], expected_url)
+
+ # Test returned copyright
+
+
From 68bb26b007425eb63eb5fabdae222f160a346d76 Mon Sep 17 00:00:00 2001
From: brianthomas
Date: Fri, 24 Mar 2017 22:47:47 -0400
Subject: [PATCH 017/146] fix parsing, add more test cases
---
apod/service.py | 21 ++-----
apod/utility.py | 64 ++++++++++++++++-----
tests/apod/test_utility.py | 111 +++++++++++++++++++++++++++++++------
3 files changed, 148 insertions(+), 48 deletions(-)
diff --git a/apod/service.py b/apod/service.py
index 7aba614..b6d03bc 100644
--- a/apod/service.py
+++ b/apod/service.py
@@ -74,27 +74,16 @@ def _apod_handler(dt, use_concept_tags=False, use_default_today_date=False):
"""Accepts a parameter dictionary. Returns the response object to be
served through the API."""
try:
- d = {}
- explanation, title, copyrght, url, hdurl, media_type = parse_apod(dt, use_default_today_date)
- LOG.debug("managed to get apod characteristics")
+ page_props = parse_apod(dt, use_default_today_date)
+ LOG.debug("managed to get apod page characteristics")
- d['explanation'] = explanation
- d['title'] = title
- d['url'] = url
- if hdurl:
- d['hdurl'] = hdurl
- d['media_type'] = media_type
-
- if copyrght:
- d['copyright'] = copyrght
-
if use_concept_tags:
if ALCHEMY_API_KEY == None:
- d['concepts'] = "concept_tags functionality turned off in current service"
+ page_props['concepts'] = "concept_tags functionality turned off in current service"
else:
- d['concepts'] = get_concepts(request, explanation, ALCHEMY_API_KEY)
+ page_props['concepts'] = get_concepts(request, page_props['explanation'], ALCHEMY_API_KEY)
- return d
+ return page_props
except Exception as e:
diff --git a/apod/utility.py b/apod/utility.py
index 03a70a2..cf30f60 100644
--- a/apod/utility.py
+++ b/apod/utility.py
@@ -44,8 +44,20 @@ def _get_apod_chars(dt):
media_type = 'video'
data = soup.iframe['src']
- return _explanation(soup), _title(soup), _copyright(soup), data, hd_data, media_type
+
+ props = {}
+
+ props['explanation'] = _explanation(soup)
+ props['title'] = _title(soup)
+ props['copyright'] = _copyright(soup)
+ props['media_type'] = media_type
+ props['url'] = data
+
+ if hd_data:
+ props['hdurl'] = hd_data
+ return props
+
def _title(soup):
"""Accepts a BeautifulSoup object for the APOD HTML page and returns the
APOD image title. Highly idiosyncratic with adaptations for different
@@ -73,21 +85,44 @@ def _copyright(soup):
# There's no uniform handling of copyright (sigh). Well, we just have to
# try every stinking text block we find...
-
- for element in soup.findAll('b', text=True):
+
+ copyright = None
+ use_next = False
+ for element in soup.findAll('a', text=True):
#LOG.debug("TEXT: "+element.text)
- # search text for explicit match
+
+ if use_next:
+ copyright = element.text.strip(' ')
+ break
+
if "Copyright" in element.text:
LOG.debug("Found Copyright text:"+str(element.text))
- LOG.debug(" element:"+str(element))
- # pull the copyright from the link text
- link_selection = element.parent.find_all('a')[0]
- if "Copyright" in link_selection.text:
- # hmm. older style, try to grab from 2nd link
- LOG.debug("trying olderstyle copyright grab")
- link_selection = element.parent.find_all('a')[1]
- # return
- return link_selection.text.strip(' ')
+ use_next = True
+
+
+ if not copyright:
+
+ for element in soup.findAll(['b','a'], text=True):
+ #LOG.debug("TEXT: "+element.text)
+ # search text for explicit match
+ if "Copyright" in element.text:
+ LOG.debug("Found Copyright text:"+str(element.text))
+ # pull the copyright from the link text
+ # which follows
+ sibling = element.next_sibling
+ stuff = ""
+ while (sibling):
+ try:
+ stuff = stuff + sibling.text
+ except Exception:
+ pass
+ sibling = sibling.next_sibling
+
+ if stuff:
+ copyright = stuff.strip(' ')
+
+
+ return copyright
except Exception as ex:
LOG.error(str(ex))
@@ -123,9 +158,8 @@ def parse_apod (dt, use_default_today_date=False):
LOG.debug("apod chars called date:"+str(dt))
try:
-
return _get_apod_chars(dt)
-
+
except Exception as ex:
# handle edge case where the service local time
diff --git a/tests/apod/test_utility.py b/tests/apod/test_utility.py
index 704f0ef..5df1c99 100644
--- a/tests/apod/test_utility.py
+++ b/tests/apod/test_utility.py
@@ -5,29 +5,106 @@
logging.basicConfig(level=logging.DEBUG)
+from datetime import datetime
class TestApod(unittest.TestCase):
-
"""Test the extraction of APOD characteristics."""
- def setUp(self):
- from datetime import datetime
- self.date = datetime (2013, 6, 13)
+
+ TEST_DATA = {
+ 'normal page, copyright' :
+ {
+ "datetime": datetime(2017, 3, 22),
+ "copyright": 'Robert Gendler',
+ "date": "2017-03-22",
+ "explanation": "In cosmic brush strokes of glowing hydrogen gas, this beautiful skyscape unfolds across the plane of our Milky Way Galaxy near the northern end of the Great Rift and the center of the constellation Cygnus the Swan. A 36 panel mosaic of telescopic image data, the scene spans about six degrees. Bright supergiant star Gamma Cygni (Sadr) to the upper left of the image center lies in the foreground of the complex gas and dust clouds and crowded star fields. Left of Gamma Cygni, shaped like two luminous wings divided by a long dark dust lane is IC 1318 whose popular name is understandably the Butterfly Nebula. The more compact, bright nebula at the lower right is NGC 6888, the Crescent Nebula. Some distance estimates for Gamma Cygni place it at around 1,800 light-years while estimates for IC 1318 and NGC 6888 range from 2,000 to 5,000 light-years.",
+ "hdurl": "http://apod.nasa.gov/apod/image/1703/Cygnus-New-L.jpg",
+ "media_type": "image",
+ "service_version": "v1",
+ "title": "Central Cygnus Skyscape",
+ "url": "http://apod.nasa.gov/apod/image/1703/Cygnus-New-1024.jpg",
+ },
+ 'newer page, Reprocessing & copyright' :
+ {
+ "datetime": datetime(2017, 2, 8),
+ "copyright": "Jesús M.Vargas & Maritxu Poyal",
+ "date": "2017-02-08",
+ "explanation": "The bright clusters and nebulae of planet Earth's night sky are often named for flowers or insects. Though its wingspan covers over 3 light-years, NGC 6302 is no exception. With an estimated surface temperature of about 250,000 degrees C, the dying central star of this particular planetary nebula has become exceptionally hot, shining brightly in ultraviolet light but hidden from direct view by a dense torus of dust. This sharp close-up of the dying star's nebula was recorded by the Hubble Space Telescope and is presented here in reprocessed colors. Cutting across a bright cavity of ionized gas, the dust torus surrounding the central star is near the center of this view, almost edge-on to the line-of-sight. Molecular hydrogen has been detected in the hot star's dusty cosmic shroud. NGC 6302 lies about 4,000 light-years away in the arachnologically correct constellation of the Scorpion (Scorpius). Follow APOD on: Facebook, Google Plus, Instagram, or Twitter",
+ "hdurl": "http://apod.nasa.gov/apod/image/1702/Butterfly_HubbleVargas_5075.jpg",
+ "media_type": "image",
+ "service_version": "v1",
+ "title": "The Butterfly Nebula from Hubble",
+ "url": "http://apod.nasa.gov/apod/image/1702/Butterfly_HubbleVargas_960.jpg"
+ },
+ 'older page, copyright' :
+ {
+ "datetime": datetime(2015, 11, 15),
+ "copyright": "Sean M. Sabatini",
+ "date": "2015-11-15",
+ "explanation": "There was a shower over Monument Valley -- but not water. Meteors. The featured image -- actually a composite of six exposures of about 30 seconds each -- was taken in 2001, a year when there was a very active Leonids shower. At that time, Earth was moving through a particularly dense swarm of sand-sized debris from Comet Tempel-Tuttle, so that meteor rates approached one visible streak per second. The meteors appear parallel because they all fall to Earth from the meteor shower radiant -- a point on the sky towards the constellation of the Lion (Leo). The yearly Leonids meteor shower peaks again this week. Although the Moon's glow should not obstruct the visibility of many meteors, this year's shower will peak with perhaps 15 meteors visible in an hour, a rate which is good but not expected to rival the 2001 Leonids. By the way -- how many meteors can you identify in the featured image?",
+ "hdurl": "http://apod.nasa.gov/apod/image/1511/leonidsmonuments_sabatini_2330.jpg",
+ "media_type": "image",
+ "service_version": "v1",
+ "title": "Leonids Over Monument Valley",
+ "url": "http://apod.nasa.gov/apod/image/1511/leonidsmonuments_sabatini_960.jpg"
+ },
+ 'older page, copyright #2' :
+ {
+ "datetime": datetime(2013, 3, 11),
+ # this illustrates problematic, but still functional parsing of the copyright
+ "copyright": 'Martin RietzeAlien Landscapes on Planet Earth',
+ "date": "2013-03-11",
+ "explanation": "Why does a volcanic eruption sometimes create lightning? Pictured above, the Sakurajima volcano in southern Japan was caught erupting in early January. Magma bubbles so hot they glow shoot away as liquid rock bursts through the Earth's surface from below. The above image is particularly notable, however, for the lightning bolts caught near the volcano's summit. Why lightning occurs even in common thunderstorms remains a topic of research, and the cause of volcanic lightning is even less clear. Surely, lightning bolts help quench areas of opposite but separated electric charges. One hypothesis holds that catapulting magma bubbles or volcanic ash are themselves electrically charged, and by their motion create these separated areas. Other volcanic lightning episodes may be facilitated by charge-inducing collisions in volcanic dust. Lightning is usually occurring somewhere on Earth, typically over 40 times each second.",
+ "hdurl": "http://apod.nasa.gov/apod/image/1303/volcano_reitze_1280.jpg",
+ "media_type": "image",
+ "service_version": "v1",
+ "title": "Sakurajima Volcano with Lightning",
+ "url": "http://apod.nasa.gov/apod/image/1303/volcano_reitze_960.jpg"
+ },
+ 'older page, no copyright' :
+ {
+ "datetime": datetime(1998, 6, 19),
+ "date": "1998-06-19",
+ "copyright": None,
+ "explanation": "Looking down on the Northern Hemisphere of Mars on June 1, the Mars Global Surveyor spacecraft's wide angle camera recorded this morning image of the red planet. Mars Global Surveyor's orbit is now oriented to view the planet's surface during the morning hours and the night/day shadow boundary or terminator arcs across the left side of the picture. Two large volcanos, Olympus Mons (left of center) and Ascraeus Mons (lower right) peer upward through seasonal haze and water-ice clouds of the Northern Martian Winter. The color image was synthesized from red and blue band pictures and only approximates a \"true color\" picture of Mars.",
+ "hdurl": "http://apod.nasa.gov/apod/image/9806/tharsis_mgs_big.jpg",
+ "media_type": "image",
+ "service_version": "v1",
+ "title": "Good Morning Mars",
+ "url": "http://apod.nasa.gov/apod/image/9806/tharsis_mgs.jpg"
+ },
+ 'older page, no copyright, #2' :
+ {
+ "datetime": datetime(2012, 8, 30),
+ "date": "2012-08-30",
+ "copyright": None,
+ "explanation": "Have you seen a panorama from another world lately? Assembled from high-resolution scans of the original film frames, this one sweeps across the magnificent desolation of the Apollo 11 landing site on the Moon's Sea of Tranquility. Taken by Neil Armstrong looking out his window of the Eagle Lunar Module, the frame at the far left (AS11-37-5449) is the first picture taken by a person on another world. Toward the south, thruster nozzles can be seen in the foreground on the left, while at the right, the shadow of the Eagle is visible toward the west. For scale, the large, shallow crater on the right has a diameter of about 12 meters. Frames taken from the Lunar Module windows about an hour and a half after landing, before walking on the lunar surface, were intended to initially document the landing site in case an early departure was necessary.",
+ "hdurl": "http://apod.nasa.gov/apod/image/1208/a11pan1040226lftsm.jpg",
+ "media_type": "image",
+ "service_version": "v1",
+ "title": "Apollo 11 Landing Site Panorama",
+ "url": "http://apod.nasa.gov/apod/image/1208/a11pan1040226lftsm600.jpg"
+ },
+ }
+
+ def _test_harness(self, test_title, data):
+
+ print ("Testing "+test_title)
+
+ # make the call
+ values = utility.parse_apod(data['datetime'])
+ # Test returned properties
+ for prop in values.keys():
+ self.assertEqual(values[prop], data[prop], "Test of property: "+prop)
+
+
def test_apod_characteristics(self):
- #explanation, title, url
- values = utility.parse_apod(self.date)
-
- # Test returned Explanation
- expected_explan = 'You can see four planets in this serene'
- self.assertTrue(values[0].startswith(expected_explan))
- # Test returned Title
- expected_title = 'Four Planet Sunset'
- self.assertEqual(values[1], expected_title)
+ for page_type in TestApod.TEST_DATA.keys():
+ self._test_harness(page_type, TestApod.TEST_DATA[page_type])
+
+
+
- # Test returned url
- expected_url = None
- self.assertEqual(values[2], expected_url)
- # Test returned copyright
From 4ba8b66771735e97a3505e09f7d193cc1280f4fe Mon Sep 17 00:00:00 2001
From: Brian Thomas
Date: Mon, 3 Apr 2017 12:51:31 -0400
Subject: [PATCH 018/146] use https instead of http
---
apod/utility.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/apod/utility.py b/apod/utility.py
index cf30f60..0d959cb 100644
--- a/apod/utility.py
+++ b/apod/utility.py
@@ -17,7 +17,7 @@
#LOG.setLevel(logging.DEBUG)
# location of backing APOD service
-BASE = 'http://apod.nasa.gov/apod/'
+BASE = 'https://apod.nasa.gov/apod/'
def _get_apod_chars(dt):
From c6410a1ec5df455c1272778442d3dbac837d7d32 Mon Sep 17 00:00:00 2001
From: Brian Thomas
Date: Mon, 3 Apr 2017 13:21:20 -0400
Subject: [PATCH 019/146] add encoding for test file so we work with odd author
name
---
tests/apod/test_utility.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/tests/apod/test_utility.py b/tests/apod/test_utility.py
index 5df1c99..e265aa0 100644
--- a/tests/apod/test_utility.py
+++ b/tests/apod/test_utility.py
@@ -1,4 +1,5 @@
-
+#!/bin/sh/python
+# coding= utf-8
import unittest
from apod import utility
import logging
From 85888599dc33280d8e1b176afa4645688437d0c6 Mon Sep 17 00:00:00 2001
From: Brian Thomas
Date: Mon, 3 Apr 2017 13:27:59 -0400
Subject: [PATCH 020/146] fix tests for http->https fix
---
tests/apod/test_utility.py | 29 ++++++++++++++---------------
1 file changed, 14 insertions(+), 15 deletions(-)
diff --git a/tests/apod/test_utility.py b/tests/apod/test_utility.py
index e265aa0..77b3407 100644
--- a/tests/apod/test_utility.py
+++ b/tests/apod/test_utility.py
@@ -17,11 +17,11 @@ class TestApod(unittest.TestCase):
"copyright": 'Robert Gendler',
"date": "2017-03-22",
"explanation": "In cosmic brush strokes of glowing hydrogen gas, this beautiful skyscape unfolds across the plane of our Milky Way Galaxy near the northern end of the Great Rift and the center of the constellation Cygnus the Swan. A 36 panel mosaic of telescopic image data, the scene spans about six degrees. Bright supergiant star Gamma Cygni (Sadr) to the upper left of the image center lies in the foreground of the complex gas and dust clouds and crowded star fields. Left of Gamma Cygni, shaped like two luminous wings divided by a long dark dust lane is IC 1318 whose popular name is understandably the Butterfly Nebula. The more compact, bright nebula at the lower right is NGC 6888, the Crescent Nebula. Some distance estimates for Gamma Cygni place it at around 1,800 light-years while estimates for IC 1318 and NGC 6888 range from 2,000 to 5,000 light-years.",
- "hdurl": "http://apod.nasa.gov/apod/image/1703/Cygnus-New-L.jpg",
+ "hdurl": "https://apod.nasa.gov/apod/image/1703/Cygnus-New-L.jpg",
"media_type": "image",
"service_version": "v1",
"title": "Central Cygnus Skyscape",
- "url": "http://apod.nasa.gov/apod/image/1703/Cygnus-New-1024.jpg",
+ "url": "https://apod.nasa.gov/apod/image/1703/Cygnus-New-1024.jpg",
},
'newer page, Reprocessing & copyright' :
{
@@ -29,11 +29,11 @@ class TestApod(unittest.TestCase):
"copyright": "Jesús M.Vargas & Maritxu Poyal",
"date": "2017-02-08",
"explanation": "The bright clusters and nebulae of planet Earth's night sky are often named for flowers or insects. Though its wingspan covers over 3 light-years, NGC 6302 is no exception. With an estimated surface temperature of about 250,000 degrees C, the dying central star of this particular planetary nebula has become exceptionally hot, shining brightly in ultraviolet light but hidden from direct view by a dense torus of dust. This sharp close-up of the dying star's nebula was recorded by the Hubble Space Telescope and is presented here in reprocessed colors. Cutting across a bright cavity of ionized gas, the dust torus surrounding the central star is near the center of this view, almost edge-on to the line-of-sight. Molecular hydrogen has been detected in the hot star's dusty cosmic shroud. NGC 6302 lies about 4,000 light-years away in the arachnologically correct constellation of the Scorpion (Scorpius). Follow APOD on: Facebook, Google Plus, Instagram, or Twitter",
- "hdurl": "http://apod.nasa.gov/apod/image/1702/Butterfly_HubbleVargas_5075.jpg",
+ "hdurl": "https://apod.nasa.gov/apod/image/1702/Butterfly_HubbleVargas_5075.jpg",
"media_type": "image",
"service_version": "v1",
"title": "The Butterfly Nebula from Hubble",
- "url": "http://apod.nasa.gov/apod/image/1702/Butterfly_HubbleVargas_960.jpg"
+ "url": "https://apod.nasa.gov/apod/image/1702/Butterfly_HubbleVargas_960.jpg"
},
'older page, copyright' :
{
@@ -41,11 +41,11 @@ class TestApod(unittest.TestCase):
"copyright": "Sean M. Sabatini",
"date": "2015-11-15",
"explanation": "There was a shower over Monument Valley -- but not water. Meteors. The featured image -- actually a composite of six exposures of about 30 seconds each -- was taken in 2001, a year when there was a very active Leonids shower. At that time, Earth was moving through a particularly dense swarm of sand-sized debris from Comet Tempel-Tuttle, so that meteor rates approached one visible streak per second. The meteors appear parallel because they all fall to Earth from the meteor shower radiant -- a point on the sky towards the constellation of the Lion (Leo). The yearly Leonids meteor shower peaks again this week. Although the Moon's glow should not obstruct the visibility of many meteors, this year's shower will peak with perhaps 15 meteors visible in an hour, a rate which is good but not expected to rival the 2001 Leonids. By the way -- how many meteors can you identify in the featured image?",
- "hdurl": "http://apod.nasa.gov/apod/image/1511/leonidsmonuments_sabatini_2330.jpg",
+ "hdurl": "https://apod.nasa.gov/apod/image/1511/leonidsmonuments_sabatini_2330.jpg",
"media_type": "image",
"service_version": "v1",
"title": "Leonids Over Monument Valley",
- "url": "http://apod.nasa.gov/apod/image/1511/leonidsmonuments_sabatini_960.jpg"
+ "url": "https://apod.nasa.gov/apod/image/1511/leonidsmonuments_sabatini_960.jpg"
},
'older page, copyright #2' :
{
@@ -54,11 +54,11 @@ class TestApod(unittest.TestCase):
"copyright": 'Martin RietzeAlien Landscapes on Planet Earth',
"date": "2013-03-11",
"explanation": "Why does a volcanic eruption sometimes create lightning? Pictured above, the Sakurajima volcano in southern Japan was caught erupting in early January. Magma bubbles so hot they glow shoot away as liquid rock bursts through the Earth's surface from below. The above image is particularly notable, however, for the lightning bolts caught near the volcano's summit. Why lightning occurs even in common thunderstorms remains a topic of research, and the cause of volcanic lightning is even less clear. Surely, lightning bolts help quench areas of opposite but separated electric charges. One hypothesis holds that catapulting magma bubbles or volcanic ash are themselves electrically charged, and by their motion create these separated areas. Other volcanic lightning episodes may be facilitated by charge-inducing collisions in volcanic dust. Lightning is usually occurring somewhere on Earth, typically over 40 times each second.",
- "hdurl": "http://apod.nasa.gov/apod/image/1303/volcano_reitze_1280.jpg",
+ "hdurl": "https://apod.nasa.gov/apod/image/1303/volcano_reitze_1280.jpg",
"media_type": "image",
"service_version": "v1",
"title": "Sakurajima Volcano with Lightning",
- "url": "http://apod.nasa.gov/apod/image/1303/volcano_reitze_960.jpg"
+ "url": "https://apod.nasa.gov/apod/image/1303/volcano_reitze_960.jpg"
},
'older page, no copyright' :
{
@@ -66,11 +66,11 @@ class TestApod(unittest.TestCase):
"date": "1998-06-19",
"copyright": None,
"explanation": "Looking down on the Northern Hemisphere of Mars on June 1, the Mars Global Surveyor spacecraft's wide angle camera recorded this morning image of the red planet. Mars Global Surveyor's orbit is now oriented to view the planet's surface during the morning hours and the night/day shadow boundary or terminator arcs across the left side of the picture. Two large volcanos, Olympus Mons (left of center) and Ascraeus Mons (lower right) peer upward through seasonal haze and water-ice clouds of the Northern Martian Winter. The color image was synthesized from red and blue band pictures and only approximates a \"true color\" picture of Mars.",
- "hdurl": "http://apod.nasa.gov/apod/image/9806/tharsis_mgs_big.jpg",
+ "hdurl": "https://apod.nasa.gov/apod/image/9806/tharsis_mgs_big.jpg",
"media_type": "image",
"service_version": "v1",
"title": "Good Morning Mars",
- "url": "http://apod.nasa.gov/apod/image/9806/tharsis_mgs.jpg"
+ "url": "https://apod.nasa.gov/apod/image/9806/tharsis_mgs.jpg"
},
'older page, no copyright, #2' :
{
@@ -78,11 +78,11 @@ class TestApod(unittest.TestCase):
"date": "2012-08-30",
"copyright": None,
"explanation": "Have you seen a panorama from another world lately? Assembled from high-resolution scans of the original film frames, this one sweeps across the magnificent desolation of the Apollo 11 landing site on the Moon's Sea of Tranquility. Taken by Neil Armstrong looking out his window of the Eagle Lunar Module, the frame at the far left (AS11-37-5449) is the first picture taken by a person on another world. Toward the south, thruster nozzles can be seen in the foreground on the left, while at the right, the shadow of the Eagle is visible toward the west. For scale, the large, shallow crater on the right has a diameter of about 12 meters. Frames taken from the Lunar Module windows about an hour and a half after landing, before walking on the lunar surface, were intended to initially document the landing site in case an early departure was necessary.",
- "hdurl": "http://apod.nasa.gov/apod/image/1208/a11pan1040226lftsm.jpg",
+ "hdurl": "https://apod.nasa.gov/apod/image/1208/a11pan1040226lftsm.jpg",
"media_type": "image",
"service_version": "v1",
"title": "Apollo 11 Landing Site Panorama",
- "url": "http://apod.nasa.gov/apod/image/1208/a11pan1040226lftsm600.jpg"
+ "url": "https://apod.nasa.gov/apod/image/1208/a11pan1040226lftsm600.jpg"
},
}
@@ -95,6 +95,8 @@ def _test_harness(self, test_title, data):
# Test returned properties
for prop in values.keys():
+ if prop == "copyright":
+ print(str(values['copyright']))
self.assertEqual(values[prop], data[prop], "Test of property: "+prop)
@@ -106,6 +108,3 @@ def test_apod_characteristics(self):
-
-
-
From 59b5acbaa2a197977e61a556436a7a554fc43bac Mon Sep 17 00:00:00 2001
From: Brian Thomas
Date: Mon, 3 Apr 2017 15:34:06 -0400
Subject: [PATCH 021/146] trivial change to force heroku update
---
README.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/README.md b/README.md
index f84d1cb..19b33d5 100644
--- a/README.md
+++ b/README.md
@@ -3,6 +3,7 @@
A microservice written in Python which may be run on Google App
Engine with the [Flask micro framework](http://flask.pocoo.org).
+
## Endpoint: `//apod`
There is only one endpoint in this service which takes 2 optional fields
From 272875285c43312596f9c680cbd3c899b41d016a Mon Sep 17 00:00:00 2001
From: Brian Thomas
Date: Mon, 3 Apr 2017 15:43:57 -0400
Subject: [PATCH 022/146] fix utility path
---
apod/service.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/apod/service.py b/apod/service.py
index b6d03bc..0054065 100644
--- a/apod/service.py
+++ b/apod/service.py
@@ -11,7 +11,7 @@
from datetime import datetime
from flask import request, jsonify, render_template, Flask
from flask_cors import CORS, cross_origin
-from utility import parse_apod, get_concepts
+from apod.utility import parse_apod, get_concepts
import logging
app = Flask(__name__)
From 7deca076057be92bc3dccc2fcf859a0a347bffbf Mon Sep 17 00:00:00 2001
From: Brian Thomas
Date: Mon, 3 Apr 2017 15:47:06 -0400
Subject: [PATCH 023/146] Fix copyright showing up null when not present
---
apod/utility.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/apod/utility.py b/apod/utility.py
index 0d959cb..ded2a3d 100644
--- a/apod/utility.py
+++ b/apod/utility.py
@@ -49,7 +49,9 @@ def _get_apod_chars(dt):
props['explanation'] = _explanation(soup)
props['title'] = _title(soup)
- props['copyright'] = _copyright(soup)
+ copyright = _copyright(soup)
+ if copyright:
+ props['copyright'] = copyright
props['media_type'] = media_type
props['url'] = data
From 69d1f1c9eba39172e9909db7d69043312a760aec Mon Sep 17 00:00:00 2001
From: Jennifer Betancourt
Date: Mon, 10 Jul 2017 20:52:16 -0700
Subject: [PATCH 024/146] Update gitignore to include pycharm project files and
pycache files
---
.gitignore | 2 ++
1 file changed, 2 insertions(+)
diff --git a/.gitignore b/.gitignore
index 9c7a6fa..d5c877d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,5 @@ alchemy_api.key
lib/
!lib/README.md
.DS_Store
+.idea/
+apod/__pycache__/
From db0d29513ba11d8a63530ffa26eaa735297c87dd Mon Sep 17 00:00:00 2001
From: Jennifer Betancourt
Date: Mon, 10 Jul 2017 20:53:42 -0700
Subject: [PATCH 025/146] Updated files to follow PEP-8 style guide
---
apod/service.py | 83 +++++++++++++++++++++++++------------------------
apod/utility.py | 72 +++++++++++++++++++++---------------------
setup.py | 17 +++++-----
3 files changed, 88 insertions(+), 84 deletions(-)
diff --git a/apod/service.py b/apod/service.py
index 0054065..f1dcbd0 100644
--- a/apod/service.py
+++ b/apod/service.py
@@ -11,7 +11,7 @@
from datetime import datetime
from flask import request, jsonify, render_template, Flask
from flask_cors import CORS, cross_origin
-from apod.utility import parse_apod, get_concepts
+from apod.utility import parse_apod, get_concepts
import logging
app = Flask(__name__)
@@ -19,78 +19,81 @@
LOG = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)
-#LOG.setLevel(logging.DEBUG)
+# LOG.setLevel(logging.DEBUG)
# this should reflect both this service and the backing
# assorted libraries
-SERVICE_VERSION='v1'
-APOD_METHOD_NAME='apod'
+SERVICE_VERSION = 'v1'
+APOD_METHOD_NAME = 'apod'
ALLOWED_APOD_FIELDS = ['concept_tags', 'date', 'hd']
ALCHEMY_API_KEY = None
-
try:
with open('alchemy_api.key', 'r') as f:
ALCHEMY_API_KEY = f.read()
except:
- LOG.info ("WARNING: NO alchemy_api.key found, concept_tagging is NOT supported")
+ LOG.info("WARNING: NO alchemy_api.key found, concept_tagging is NOT supported")
-def _abort(code, msg, usage=True):
+def _abort(code, msg, usage=True):
if (usage):
- msg += " "+_usage()+"'"
+ msg += " " + _usage() + "'"
response = jsonify(service_version=SERVICE_VERSION, msg=msg, code=code)
response.status_code = code
LOG.debug(str(response))
-
+
return response
+
def _usage(joinstr="', '", prestr="'"):
- return "Allowed request fields for "+APOD_METHOD_NAME+" method are "+prestr+joinstr.join(ALLOWED_APOD_FIELDS)
+ return "Allowed request fields for " + APOD_METHOD_NAME + " method are " + prestr + joinstr.join(
+ ALLOWED_APOD_FIELDS)
-def _validate (data):
+
+def _validate(data):
LOG.debug("_validate(data) called")
for key in data:
if key not in ALLOWED_APOD_FIELDS:
return False
return True
-def _validate_date (dt):
-
+
+def _validate_date(dt):
LOG.debug("_validate_date(dt) called")
today = datetime.today()
- begin = datetime (1995, 6, 16) # first APOD image date
-
+ begin = datetime(1995, 6, 16) # first APOD image date
+
# validate input
if (dt > today) or (dt < begin):
-
today_str = today.strftime('%b %d, %Y')
begin_str = begin.strftime('%b %d, %Y')
-
+
raise ValueError('Date must be between %s and %s.' % (begin_str, today_str))
-
+
+
def _apod_handler(dt, use_concept_tags=False, use_default_today_date=False):
"""Accepts a parameter dictionary. Returns the response object to be
served through the API."""
try:
page_props = parse_apod(dt, use_default_today_date)
LOG.debug("managed to get apod page characteristics")
-
+
if use_concept_tags:
if ALCHEMY_API_KEY == None:
page_props['concepts'] = "concept_tags functionality turned off in current service"
else:
page_props['concepts'] = get_concepts(request, page_props['explanation'], ALCHEMY_API_KEY)
-
- return page_props
-
+
+ return page_props
+
except Exception as e:
-
- LOG.error("Internal Service Error :"+str(type(e))+" msg:"+str(e))
+
+ LOG.error("Internal Service Error :" + str(type(e)) + " msg:" + str(e))
# return code 500 here
return _abort(500, "Internal Service Error", usage=False)
-
+
+
#
# Endpoints
#
@@ -98,13 +101,13 @@ def _apod_handler(dt, use_concept_tags=False, use_default_today_date=False):
@app.route('/')
def home():
return render_template('home.html', version=SERVICE_VERSION, \
- service_url=request.host, \
- methodname=APOD_METHOD_NAME, \
- usage=_usage(joinstr='", "', prestr='"')+'"')
+ service_url=request.host, \
+ methodname=APOD_METHOD_NAME, \
+ usage=_usage(joinstr='", "', prestr='"') + '"')
-@app.route('/'+SERVICE_VERSION+'/'+APOD_METHOD_NAME+'/', methods=['GET'])
-def apod():
+@app.route('/' + SERVICE_VERSION + '/' + APOD_METHOD_NAME + '/', methods=['GET'])
+def apod():
LOG.info("apod path called")
try:
@@ -112,8 +115,8 @@ def apod():
args = request.args
if not _validate(args):
- return _abort (400, "Bad Request incorrect field passed.")
-
+ return _abort(400, "Bad Request incorrect field passed.")
+
# get the date param
use_default_today_date = False
date = args.get('date')
@@ -121,34 +124,35 @@ def apod():
# fall back to using today's date IF they didn't specify a date
date = datetime.strftime(datetime.today(), '%Y-%m-%d')
use_default_today_date = True
-
+
# grab the concept_tags param
use_concept_tags = args.get('concept_tags', False)
-
+
# validate input date
dt = datetime.strptime(date, '%Y-%m-%d')
_validate_date(dt)
-
+
# get data
data = _apod_handler(dt, use_concept_tags, use_default_today_date)
data['date'] = date
data['service_version'] = SERVICE_VERSION
-
+
# return info as JSON
return jsonify(data)
except ValueError as ve:
return _abort(400, str(ve), False)
-
+
except Exception as ex:
etype = type(ex)
if etype == ValueError or "BadRequest" in str(etype):
- return _abort(400, str(ex)+".")
+ return _abort(400, str(ex) + ".")
else:
- LOG.error("Service Exception. Msg: "+str(type(ex)))
+ LOG.error("Service Exception. Msg: " + str(type(ex)))
return _abort(500, "Internal Service Error", usage=False)
+
@app.errorhandler(404)
def page_not_found(e):
"""Return a custom 404 error."""
@@ -163,4 +167,3 @@ def application_error(e):
if __name__ == '__main__':
app.run()
-
diff --git a/apod/utility.py b/apod/utility.py
index ded2a3d..79e28e2 100644
--- a/apod/utility.py
+++ b/apod/utility.py
@@ -14,17 +14,17 @@
LOG = logging.getLogger(__name__)
logging.basicConfig(level=logging.WARN)
-#LOG.setLevel(logging.DEBUG)
+# LOG.setLevel(logging.DEBUG)
# location of backing APOD service
BASE = 'https://apod.nasa.gov/apod/'
+
def _get_apod_chars(dt):
-
media_type = 'image'
date_str = dt.strftime('%y%m%d')
apod_url = '%sap%s.html' % (BASE, date_str)
- LOG.debug("OPENING URL:"+apod_url)
+ LOG.debug("OPENING URL:" + apod_url)
soup = BeautifulSoup(requests.get(apod_url).text, "html.parser")
LOG.debug("getting the data url")
data = None
@@ -33,7 +33,7 @@ def _get_apod_chars(dt):
# it is an image, so get both the low- and high-resolution data
data = BASE + soup.img['src']
hd_data = data
-
+
LOG.debug("getting the link for hd_data")
for link in soup.find_all('a', href=True):
if link['href'] and link['href'].startswith("image"):
@@ -43,23 +43,23 @@ def _get_apod_chars(dt):
# its a video
media_type = 'video'
data = soup.iframe['src']
-
-
+
props = {}
-
- props['explanation'] = _explanation(soup)
- props['title'] = _title(soup)
+
+ props['explanation'] = _explanation(soup)
+ props['title'] = _title(soup)
copyright = _copyright(soup)
if copyright:
props['copyright'] = copyright
props['media_type'] = media_type
props['url'] = data
-
+
if hd_data:
props['hdurl'] = hd_data
-
+
return props
+
def _title(soup):
"""Accepts a BeautifulSoup object for the APOD HTML page and returns the
APOD image title. Highly idiosyncratic with adaptations for different
@@ -77,6 +77,7 @@ def _title(soup):
else:
raise ValueError('Unsupported schema for given date.')
+
def _copyright(soup):
"""Accepts a BeautifulSoup object for the APOD HTML page and returns the
APOD image copyright. Highly idiosyncratic with adaptations for different
@@ -87,28 +88,27 @@ def _copyright(soup):
# There's no uniform handling of copyright (sigh). Well, we just have to
# try every stinking text block we find...
-
+
copyright = None
use_next = False
for element in soup.findAll('a', text=True):
- #LOG.debug("TEXT: "+element.text)
-
+ # LOG.debug("TEXT: "+element.text)
+
if use_next:
copyright = element.text.strip(' ')
break
-
+
if "Copyright" in element.text:
- LOG.debug("Found Copyright text:"+str(element.text))
+ LOG.debug("Found Copyright text:" + str(element.text))
use_next = True
-
-
+
if not copyright:
-
- for element in soup.findAll(['b','a'], text=True):
- #LOG.debug("TEXT: "+element.text)
+
+ for element in soup.findAll(['b', 'a'], text=True):
+ # LOG.debug("TEXT: "+element.text)
# search text for explicit match
if "Copyright" in element.text:
- LOG.debug("Found Copyright text:"+str(element.text))
+ LOG.debug("Found Copyright text:" + str(element.text))
# pull the copyright from the link text
# which follows
sibling = element.next_sibling
@@ -119,11 +119,10 @@ def _copyright(soup):
except Exception:
pass
sibling = sibling.next_sibling
-
+
if stuff:
copyright = stuff.strip(' ')
-
-
+
return copyright
except Exception as ex:
@@ -133,6 +132,7 @@ def _copyright(soup):
# NO stated copyright, so we return None
return None
+
def _explanation(soup):
"""Accepts a BeautifulSoup object for the APOD HTML page and returns the
APOD image explanation. Highly idiosyncratic."""
@@ -153,22 +153,22 @@ def _explanation(soup):
return s
-def parse_apod (dt, use_default_today_date=False):
+def parse_apod(dt, use_default_today_date=False):
"""Accepts a date in '%Y-%m-%d' format. Returns the URL of the APOD image
of that day, noting that """
- LOG.debug("apod chars called date:"+str(dt))
-
+ LOG.debug("apod chars called date:" + str(dt))
+
try:
return _get_apod_chars(dt)
-
+
except Exception as ex:
-
+
# handle edge case where the service local time
# miss-matches with 'todays date' of the underlying APOD
# service (can happen because they are deployed in different
# timezones). Use the fallback of prior day's date
-
+
if use_default_today_date:
# try to get the day before
dt = dt - timedelta(days=1)
@@ -177,13 +177,13 @@ def parse_apod (dt, use_default_today_date=False):
# pass exception up the call stack
LOG.error(str(ex))
raise Exception(ex)
-
-
+
+
def get_concepts(request, text, apikey):
"""Returns the concepts associated with the text, interleaved with integer
keys indicating the index."""
cbase = 'http://access.alchemyapi.com/calls/text/TextGetRankedConcepts'
-
+
params = dict(
outputMode='json',
apikey=apikey,
@@ -191,11 +191,11 @@ def get_concepts(request, text, apikey):
)
try:
-
+
LOG.debug("Getting response")
response = json.loads(request.get(cbase, fields=params))
clist = [concept['text'] for concept in response['concepts']]
return {k: v for k, v in zip(range(len(clist)), clist)}
-
+
except Exception as ex:
raise ValueError(ex)
diff --git a/setup.py b/setup.py
index eff23bd..e5ea517 100644
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,5 @@
-
from os.path import dirname, join
-from setuptools import setup, find_packages, Command
+from setuptools import setup, find_packages, Command
with open('requirements.txt') as f:
reqs = f.read().splitlines()
@@ -23,6 +22,7 @@
try:
from setupext import janitor
+
CleanCommand = janitor.CleanCommand
except ImportError:
CleanCommand = None
@@ -35,19 +35,21 @@
long_description = f.read().decode('ascii').strip()
import os
-scripts = [os.path.join("bin",file) for file in os.listdir("bin")]
+
+scripts = [os.path.join("bin", file) for file in os.listdir("bin")]
import apod
-version=apod.version
-setup (
+version = apod.version
+
+setup(
name='apod-api',
description='Python microservice for APOD site',
url='https://www.github.com/nasa/apod-api',
version=version,
- keywords = 'apod api nasa python',
+ keywords='apod api nasa python',
long_description=long_description,
scripts=scripts,
@@ -61,9 +63,8 @@
include_package_data=True,
setup_requires=['setupext-janitor'],
- cmdclass = cmd_classes,
+ cmdclass=cmd_classes,
install_requires=reqs,
)
-
From 1412ea2e13b9fd34996100c125413f914a358921 Mon Sep 17 00:00:00 2001
From: Jennifer Betancourt
Date: Tue, 11 Jul 2017 22:19:04 -0700
Subject: [PATCH 026/146] Added API calls: 1) Return date range between start
date and end date 2) Return a set of randomly selected images
---
apod/service.py | 185 +++++++++++++++++++++++++++++++++++++++---------
apod/utility.py | 6 +-
2 files changed, 155 insertions(+), 36 deletions(-)
diff --git a/apod/service.py b/apod/service.py
index f1dcbd0..fe8dc7d 100644
--- a/apod/service.py
+++ b/apod/service.py
@@ -1,16 +1,19 @@
-''' A micro-service passing back enhanced information from Astronomy
- Picture of the Day (APOD).
+"""
+A micro-service passing back enhanced information from Astronomy
+Picture of the Day (APOD).
- Adapted from code in https://github.com/nasa/planetary-api
- Dec 1, 2015 (written by Dan Hammer)
+Adapted from code in https://github.com/nasa/planetary-api
+Dec 1, 2015 (written by Dan Hammer)
- @author=danhammer
- @author=bathomas @email=brian.a.thomas@nasa.gov
-'''
+@author=danhammer
+@author=bathomas @email=brian.a.thomas@nasa.gov
+@author=jnbetancourt @email=jennifer.n.betancourt@nasa.gov
+"""
-from datetime import datetime
+from datetime import datetime, date
+from random import sample
from flask import request, jsonify, render_template, Flask
-from flask_cors import CORS, cross_origin
+from flask_cors import CORS
from apod.utility import parse_apod, get_concepts
import logging
@@ -25,7 +28,7 @@
# assorted libraries
SERVICE_VERSION = 'v1'
APOD_METHOD_NAME = 'apod'
-ALLOWED_APOD_FIELDS = ['concept_tags', 'date', 'hd']
+ALLOWED_APOD_FIELDS = ['concept_tags', 'date', 'hd', 'count', 'start_date', 'end_date']
ALCHEMY_API_KEY = None
try:
@@ -36,7 +39,7 @@
def _abort(code, msg, usage=True):
- if (usage):
+ if usage:
msg += " " + _usage() + "'"
response = jsonify(service_version=SERVICE_VERSION, msg=msg, code=code)
@@ -59,8 +62,9 @@ def _validate(data):
return True
-def _validate_date(dt):
- LOG.debug("_validate_date(dt) called")
+# TODO(jbetancourt) Convert all datetime objects to dates, then remove this function
+def _validate_datetime(dt):
+ LOG.debug("_validate_datetime(dt) called")
today = datetime.today()
begin = datetime(1995, 6, 16) # first APOD image date
@@ -72,6 +76,19 @@ def _validate_date(dt):
raise ValueError('Date must be between %s and %s.' % (begin_str, today_str))
+def _validate_date(dt):
+ LOG.debug("_validate_date(dt) called")
+ today = datetime.today().date()
+ begin = datetime(1995, 6, 16).date() # first APOD image date
+
+ # validate input
+ if (dt > today) or (dt < begin):
+ today_str = today.strftime('%b %d, %Y')
+ begin_str = begin.strftime('%b %d, %Y')
+
+ raise ValueError('Date must be between %s and %s.' % (begin_str, today_str))
+
+
def _apod_handler(dt, use_concept_tags=False, use_default_today_date=False):
"""Accepts a parameter dictionary. Returns the response object to be
served through the API."""
@@ -94,15 +111,118 @@ def _apod_handler(dt, use_concept_tags=False, use_default_today_date=False):
return _abort(500, "Internal Service Error", usage=False)
+def _get_json_for_date(date, use_concept_tags):
+ """
+ This returns the JSON data for a specific date, which must be a string of the form YYYY-MM-DD. If date is None,
+ then it defaults to the current date.
+ :param date:
+ :param use_concept_tags:
+ :return:
+ """
+ # get the date param
+ use_default_today_date = False
+ if not date:
+ # fall back to using today's date IF they didn't specify a date
+ date = datetime.strftime(datetime.today(), '%Y-%m-%d')
+ use_default_today_date = True
+
+ # validate input date
+ dt = datetime.strptime(date, '%Y-%m-%d')
+ _validate_datetime(dt)
+
+ # get data
+ data = _apod_handler(dt, use_concept_tags, use_default_today_date)
+ data['date'] = date
+ data['service_version'] = SERVICE_VERSION
+
+ # return info as JSON
+ return jsonify(data)
+
+
+def _get_json_for_random_dates(count, use_concept_tags):
+ """
+ This returns the JSON data for a set of randomly chosen dates. The number of dates is specified by the count
+ parameter
+ :param count:
+ :param use_concept_tags:
+ :return:
+ """
+
+ if count > 100 or count <= 0:
+ raise ValueError('Count must be positive and cannot exceed 100')
+
+ begin_ordinal = datetime(1995, 6, 16).toordinal()
+ today_ordinal = datetime.today().toordinal()
+
+ date_range = range(begin_ordinal, today_ordinal + 1)
+ random_date_ordinals = sample(date_range, count)
+
+ all_data = []
+ for date_ordinal in random_date_ordinals:
+ dt = date.fromordinal(date_ordinal)
+ data = _apod_handler(datetime.combine(dt, datetime.min.time()), use_concept_tags,
+ date_ordinal == today_ordinal)
+ data['date'] = dt.isoformat()
+ data['service_version'] = SERVICE_VERSION
+ all_data.append(data)
+
+ return jsonify(all_data)
+
+
+def _get_json_for_date_range(start_date, end_date, use_concept_tags):
+ """
+ This returns the JSON data for a range of dates, specified by start_date and end_date, which must be strings of the
+ form YYYY-MM-DD. If end_date is None then it defaults to the current date.
+ :param start_date:
+ :param end_date:
+ :param use_concept_tags:
+ :return:
+ """
+ # validate input date
+ start_dt = datetime.strptime(start_date, '%Y-%m-%d').date()
+ _validate_date(start_dt)
+
+ # get the date param
+ if not end_date:
+ # fall back to using today's date IF they didn't specify a date
+ end_date = datetime.strftime(datetime.today(), '%Y-%m-%d')
+
+ # validate input date
+ end_dt = datetime.strptime(end_date, '%Y-%m-%d').date()
+ _validate_date(end_dt)
+
+ start_ordinal = start_dt.toordinal()
+ end_ordinal = end_dt.toordinal()
+ today_ordinal = datetime.today().date().toordinal()
+
+ if start_ordinal > end_ordinal:
+ raise ValueError('start_date cannot be after end_date')
+
+ all_data = []
+
+ while start_ordinal <= end_ordinal:
+ # get data
+ dt = date.fromordinal(start_ordinal)
+ data = _apod_handler(datetime.combine(dt, datetime.min.time()), use_concept_tags,
+ start_ordinal == today_ordinal)
+ data['date'] = dt.isoformat()
+ data['service_version'] = SERVICE_VERSION
+ all_data.append(data)
+ start_ordinal += 1
+
+ # return info as JSON
+ return jsonify(all_data)
+
+
#
# Endpoints
#
@app.route('/')
def home():
- return render_template('home.html', version=SERVICE_VERSION, \
- service_url=request.host, \
- methodname=APOD_METHOD_NAME, \
+ return render_template('home.html', version=SERVICE_VERSION,
+ service_url=request.host,
+ methodname=APOD_METHOD_NAME,
usage=_usage(joinstr='", "', prestr='"') + '"')
@@ -117,28 +237,27 @@ def apod():
if not _validate(args):
return _abort(400, "Bad Request incorrect field passed.")
- # get the date param
- use_default_today_date = False
+ #
date = args.get('date')
- if not date:
- # fall back to using today's date IF they didn't specify a date
- date = datetime.strftime(datetime.today(), '%Y-%m-%d')
- use_default_today_date = True
-
- # grab the concept_tags param
+ count = args.get('count')
+ start_date = args.get('start_date')
+ end_date = args.get('end_date')
use_concept_tags = args.get('concept_tags', False)
- # validate input date
- dt = datetime.strptime(date, '%Y-%m-%d')
- _validate_date(dt)
+ if not count and not start_date and not end_date:
+ return _get_json_for_date(date, use_concept_tags)
+
+ elif not date and not start_date and not end_date and count:
+ return _get_json_for_random_dates(int(count), use_concept_tags)
+
+ elif not count and not date and start_date:
+ return _get_json_for_date_range(start_date, end_date, use_concept_tags)
+
+ else:
+ return _abort(400, "Bad Request invalid field combination passed.")
+
- # get data
- data = _apod_handler(dt, use_concept_tags, use_default_today_date)
- data['date'] = date
- data['service_version'] = SERVICE_VERSION
- # return info as JSON
- return jsonify(data)
except ValueError as ve:
return _abort(400, str(ve), False)
diff --git a/apod/utility.py b/apod/utility.py
index 79e28e2..c1a604d 100644
--- a/apod/utility.py
+++ b/apod/utility.py
@@ -1,11 +1,11 @@
-'''
+"""
Split off some library functions for easier testing and code management.
Created on Mar 24, 2017
@author=bathomas @email=brian.a.thomas@nasa.gov
-'''
-
+"""
+# TODO(jbetancourt) Change double quotes to single quotes for consistency
from bs4 import BeautifulSoup
from datetime import timedelta
import requests
From 3799c055a155d2fd524ba42ba538090850b3bc03 Mon Sep 17 00:00:00 2001
From: Jennifer Betancourt
Date: Wed, 12 Jul 2017 18:36:00 -0700
Subject: [PATCH 027/146] Changed double quotes to single quotes for
consistency.
---
apod/service.py | 46 +++++++++++++++++++++----------------
apod/utility.py | 61 ++++++++++++++++++++++++++++---------------------
2 files changed, 61 insertions(+), 46 deletions(-)
diff --git a/apod/service.py b/apod/service.py
index fe8dc7d..af45876 100644
--- a/apod/service.py
+++ b/apod/service.py
@@ -35,7 +35,7 @@
with open('alchemy_api.key', 'r') as f:
ALCHEMY_API_KEY = f.read()
except:
- LOG.info("WARNING: NO alchemy_api.key found, concept_tagging is NOT supported")
+ LOG.info('WARNING: NO alchemy_api.key found, concept_tagging is NOT supported')
def _abort(code, msg, usage=True):
@@ -50,12 +50,12 @@ def _abort(code, msg, usage=True):
def _usage(joinstr="', '", prestr="'"):
- return "Allowed request fields for " + APOD_METHOD_NAME + " method are " + prestr + joinstr.join(
+ return 'Allowed request fields for ' + APOD_METHOD_NAME + ' method are ' + prestr + joinstr.join(
ALLOWED_APOD_FIELDS)
def _validate(data):
- LOG.debug("_validate(data) called")
+ LOG.debug('_validate(data) called')
for key in data:
if key not in ALLOWED_APOD_FIELDS:
return False
@@ -64,7 +64,7 @@ def _validate(data):
# TODO(jbetancourt) Convert all datetime objects to dates, then remove this function
def _validate_datetime(dt):
- LOG.debug("_validate_datetime(dt) called")
+ LOG.debug('_validate_datetime(dt) called')
today = datetime.today()
begin = datetime(1995, 6, 16) # first APOD image date
@@ -77,7 +77,7 @@ def _validate_datetime(dt):
def _validate_date(dt):
- LOG.debug("_validate_date(dt) called")
+ LOG.debug('_validate_date(dt) called')
today = datetime.today().date()
begin = datetime(1995, 6, 16).date() # first APOD image date
@@ -90,15 +90,17 @@ def _validate_date(dt):
def _apod_handler(dt, use_concept_tags=False, use_default_today_date=False):
- """Accepts a parameter dictionary. Returns the response object to be
- served through the API."""
+ """
+ Accepts a parameter dictionary. Returns the response object to be
+ served through the API.
+ """
try:
page_props = parse_apod(dt, use_default_today_date)
- LOG.debug("managed to get apod page characteristics")
+ LOG.debug('managed to get apod page characteristics')
if use_concept_tags:
if ALCHEMY_API_KEY == None:
- page_props['concepts'] = "concept_tags functionality turned off in current service"
+ page_props['concepts'] = 'concept_tags functionality turned off in current service'
else:
page_props['concepts'] = get_concepts(request, page_props['explanation'], ALCHEMY_API_KEY)
@@ -106,9 +108,9 @@ def _apod_handler(dt, use_concept_tags=False, use_default_today_date=False):
except Exception as e:
- LOG.error("Internal Service Error :" + str(type(e)) + " msg:" + str(e))
+ LOG.error('Internal Service Error :' + str(type(e)) + ' msg:' + str(e))
# return code 500 here
- return _abort(500, "Internal Service Error", usage=False)
+ return _abort(500, 'Internal Service Error', usage=False)
def _get_json_for_date(date, use_concept_tags):
@@ -228,14 +230,14 @@ def home():
@app.route('/' + SERVICE_VERSION + '/' + APOD_METHOD_NAME + '/', methods=['GET'])
def apod():
- LOG.info("apod path called")
+ LOG.info('apod path called')
try:
# application/json GET method
args = request.args
if not _validate(args):
- return _abort(400, "Bad Request incorrect field passed.")
+ return _abort(400, 'Bad Request incorrect field passed.')
#
date = args.get('date')
@@ -254,7 +256,7 @@ def apod():
return _get_json_for_date_range(start_date, end_date, use_concept_tags)
else:
- return _abort(400, "Bad Request invalid field combination passed.")
+ return _abort(400, 'Bad Request invalid field combination passed.')
@@ -265,22 +267,26 @@ def apod():
except Exception as ex:
etype = type(ex)
- if etype == ValueError or "BadRequest" in str(etype):
+ if etype == ValueError or 'BadRequest' in str(etype):
return _abort(400, str(ex) + ".")
else:
- LOG.error("Service Exception. Msg: " + str(type(ex)))
- return _abort(500, "Internal Service Error", usage=False)
+ LOG.error('Service Exception. Msg: ' + str(type(ex)))
+ return _abort(500, 'Internal Service Error', usage=False)
@app.errorhandler(404)
def page_not_found(e):
- """Return a custom 404 error."""
- return _abort(404, "Sorry, Nothing at this URL.", usage=True)
+ """
+ Return a custom 404 error.
+ """
+ return _abort(404, 'Sorry, Nothing at this URL.', usage=True)
@app.errorhandler(500)
def application_error(e):
- """Return a custom 500 error."""
+ """
+ Return a custom 500 error.
+ """
return _abort('Sorry, unexpected error: {}'.format(e), usage=False)
diff --git a/apod/utility.py b/apod/utility.py
index c1a604d..20aae6c 100644
--- a/apod/utility.py
+++ b/apod/utility.py
@@ -5,7 +5,7 @@
@author=bathomas @email=brian.a.thomas@nasa.gov
"""
-# TODO(jbetancourt) Change double quotes to single quotes for consistency
+
from bs4 import BeautifulSoup
from datetime import timedelta
import requests
@@ -24,9 +24,9 @@ def _get_apod_chars(dt):
media_type = 'image'
date_str = dt.strftime('%y%m%d')
apod_url = '%sap%s.html' % (BASE, date_str)
- LOG.debug("OPENING URL:" + apod_url)
- soup = BeautifulSoup(requests.get(apod_url).text, "html.parser")
- LOG.debug("getting the data url")
+ LOG.debug('OPENING URL:' + apod_url)
+ soup = BeautifulSoup(requests.get(apod_url).text, 'html.parser')
+ LOG.debug('getting the data url')
data = None
hd_data = None
if soup.img:
@@ -34,9 +34,9 @@ def _get_apod_chars(dt):
data = BASE + soup.img['src']
hd_data = data
- LOG.debug("getting the link for hd_data")
+ LOG.debug('getting the link for hd_data')
for link in soup.find_all('a', href=True):
- if link['href'] and link['href'].startswith("image"):
+ if link['href'] and link['href'].startswith('image'):
hd_data = BASE + link['href']
break
else:
@@ -61,10 +61,12 @@ def _get_apod_chars(dt):
def _title(soup):
- """Accepts a BeautifulSoup object for the APOD HTML page and returns the
+ """
+ Accepts a BeautifulSoup object for the APOD HTML page and returns the
APOD image title. Highly idiosyncratic with adaptations for different
- HTML structures that appear over time."""
- LOG.debug("getting the title")
+ HTML structures that appear over time.
+ """
+ LOG.debug('getting the title')
try:
# Handler for later APOD entries
center_selection = soup.find_all('center')[1]
@@ -79,13 +81,14 @@ def _title(soup):
def _copyright(soup):
- """Accepts a BeautifulSoup object for the APOD HTML page and returns the
+ """
+ Accepts a BeautifulSoup object for the APOD HTML page and returns the
APOD image copyright. Highly idiosyncratic with adaptations for different
- HTML structures that appear over time."""
- LOG.debug("getting the copyright")
+ HTML structures that appear over time.
+ """
+ LOG.debug('getting the copyright')
try:
# Handler for later APOD entries
-
# There's no uniform handling of copyright (sigh). Well, we just have to
# try every stinking text block we find...
@@ -98,8 +101,8 @@ def _copyright(soup):
copyright = element.text.strip(' ')
break
- if "Copyright" in element.text:
- LOG.debug("Found Copyright text:" + str(element.text))
+ if 'Copyright' in element.text:
+ LOG.debug('Found Copyright text:' + str(element.text))
use_next = True
if not copyright:
@@ -107,8 +110,8 @@ def _copyright(soup):
for element in soup.findAll(['b', 'a'], text=True):
# LOG.debug("TEXT: "+element.text)
# search text for explicit match
- if "Copyright" in element.text:
- LOG.debug("Found Copyright text:" + str(element.text))
+ if 'Copyright' in element.text:
+ LOG.debug('Found Copyright text:' + str(element.text))
# pull the copyright from the link text
# which follows
sibling = element.next_sibling
@@ -134,10 +137,12 @@ def _copyright(soup):
def _explanation(soup):
- """Accepts a BeautifulSoup object for the APOD HTML page and returns the
- APOD image explanation. Highly idiosyncratic."""
+ """
+ Accepts a BeautifulSoup object for the APOD HTML page and returns the
+ APOD image explanation. Highly idiosyncratic.
+ """
# Handler for later APOD entries
- LOG.debug("getting the explanation")
+ LOG.debug('getting the explanation')
s = soup.find_all('p')[2].text
s = s.replace('\n', ' ')
s = s.replace(' ', ' ')
@@ -154,10 +159,12 @@ def _explanation(soup):
def parse_apod(dt, use_default_today_date=False):
- """Accepts a date in '%Y-%m-%d' format. Returns the URL of the APOD image
- of that day, noting that """
+ """
+ Accepts a date in '%Y-%m-%d' format. Returns the URL of the APOD image
+ of that day, noting that
+ """
- LOG.debug("apod chars called date:" + str(dt))
+ LOG.debug('apod chars called date:' + str(dt))
try:
return _get_apod_chars(dt)
@@ -180,8 +187,10 @@ def parse_apod(dt, use_default_today_date=False):
def get_concepts(request, text, apikey):
- """Returns the concepts associated with the text, interleaved with integer
- keys indicating the index."""
+ """
+ Returns the concepts associated with the text, interleaved with integer
+ keys indicating the index.
+ """
cbase = 'http://access.alchemyapi.com/calls/text/TextGetRankedConcepts'
params = dict(
@@ -192,7 +201,7 @@ def get_concepts(request, text, apikey):
try:
- LOG.debug("Getting response")
+ LOG.debug('Getting response')
response = json.loads(request.get(cbase, fields=params))
clist = [concept['text'] for concept in response['concepts']]
return {k: v for k, v in zip(range(len(clist)), clist)}
From fc4471b8c4d90d26c48ce9ccf8f6872d141dc65f Mon Sep 17 00:00:00 2001
From: Jennifer Betancourt
Date: Wed, 12 Jul 2017 19:05:07 -0700
Subject: [PATCH 028/146] Fixed pylint warnings
---
apod/service.py | 35 +++++++++++++++++------------------
apod/utility.py | 24 ++++++++++--------------
2 files changed, 27 insertions(+), 32 deletions(-)
diff --git a/apod/service.py b/apod/service.py
index af45876..457d88d 100644
--- a/apod/service.py
+++ b/apod/service.py
@@ -34,7 +34,7 @@
try:
with open('alchemy_api.key', 'r') as f:
ALCHEMY_API_KEY = f.read()
-except:
+except FileNotFoundError:
LOG.info('WARNING: NO alchemy_api.key found, concept_tagging is NOT supported')
@@ -99,7 +99,7 @@ def _apod_handler(dt, use_concept_tags=False, use_default_today_date=False):
LOG.debug('managed to get apod page characteristics')
if use_concept_tags:
- if ALCHEMY_API_KEY == None:
+ if ALCHEMY_API_KEY is None:
page_props['concepts'] = 'concept_tags functionality turned off in current service'
else:
page_props['concepts'] = get_concepts(request, page_props['explanation'], ALCHEMY_API_KEY)
@@ -113,28 +113,29 @@ def _apod_handler(dt, use_concept_tags=False, use_default_today_date=False):
return _abort(500, 'Internal Service Error', usage=False)
-def _get_json_for_date(date, use_concept_tags):
+def _get_json_for_date(input_date, use_concept_tags):
"""
This returns the JSON data for a specific date, which must be a string of the form YYYY-MM-DD. If date is None,
then it defaults to the current date.
- :param date:
+ :param input_date:
:param use_concept_tags:
:return:
"""
+
# get the date param
use_default_today_date = False
- if not date:
+ if not input_date:
# fall back to using today's date IF they didn't specify a date
- date = datetime.strftime(datetime.today(), '%Y-%m-%d')
+ input_date = datetime.strftime(datetime.today(), '%Y-%m-%d')
use_default_today_date = True
# validate input date
- dt = datetime.strptime(date, '%Y-%m-%d')
+ dt = datetime.strptime(input_date, '%Y-%m-%d')
_validate_datetime(dt)
# get data
data = _apod_handler(dt, use_concept_tags, use_default_today_date)
- data['date'] = date
+ data['date'] = input_date
data['service_version'] = SERVICE_VERSION
# return info as JSON
@@ -237,29 +238,26 @@ def apod():
args = request.args
if not _validate(args):
- return _abort(400, 'Bad Request incorrect field passed.')
+ return _abort(400, 'Bad Request: incorrect field passed.')
#
- date = args.get('date')
+ input_date = args.get('date')
count = args.get('count')
start_date = args.get('start_date')
end_date = args.get('end_date')
use_concept_tags = args.get('concept_tags', False)
if not count and not start_date and not end_date:
- return _get_json_for_date(date, use_concept_tags)
+ return _get_json_for_date(input_date, use_concept_tags)
- elif not date and not start_date and not end_date and count:
+ elif not input_date and not start_date and not end_date and count:
return _get_json_for_random_dates(int(count), use_concept_tags)
- elif not count and not date and start_date:
+ elif not count and not input_date and start_date:
return _get_json_for_date_range(start_date, end_date, use_concept_tags)
else:
- return _abort(400, 'Bad Request invalid field combination passed.')
-
-
-
+ return _abort(400, 'Bad Request: invalid field combination passed.')
except ValueError as ve:
return _abort(400, str(ve), False)
@@ -279,6 +277,7 @@ def page_not_found(e):
"""
Return a custom 404 error.
"""
+ LOG.info('Invalid page request: ' + e)
return _abort(404, 'Sorry, Nothing at this URL.', usage=True)
@@ -287,7 +286,7 @@ def application_error(e):
"""
Return a custom 500 error.
"""
- return _abort('Sorry, unexpected error: {}'.format(e), usage=False)
+ return _abort(500, 'Sorry, unexpected error: {}'.format(e), usage=False)
if __name__ == '__main__':
diff --git a/apod/utility.py b/apod/utility.py
index 20aae6c..bc4a5cf 100644
--- a/apod/utility.py
+++ b/apod/utility.py
@@ -27,7 +27,6 @@ def _get_apod_chars(dt):
LOG.debug('OPENING URL:' + apod_url)
soup = BeautifulSoup(requests.get(apod_url).text, 'html.parser')
LOG.debug('getting the data url')
- data = None
hd_data = None
if soup.img:
# it is an image, so get both the low- and high-resolution data
@@ -48,9 +47,9 @@ def _get_apod_chars(dt):
props['explanation'] = _explanation(soup)
props['title'] = _title(soup)
- copyright = _copyright(soup)
- if copyright:
- props['copyright'] = copyright
+ copyright_text = _copyright(soup)
+ if copyright_text:
+ props['copyright'] = copyright_text
props['media_type'] = media_type
props['url'] = data
@@ -92,20 +91,20 @@ def _copyright(soup):
# There's no uniform handling of copyright (sigh). Well, we just have to
# try every stinking text block we find...
- copyright = None
+ copyright_text = None
use_next = False
for element in soup.findAll('a', text=True):
# LOG.debug("TEXT: "+element.text)
if use_next:
- copyright = element.text.strip(' ')
+ copyright_text = element.text.strip(' ')
break
if 'Copyright' in element.text:
LOG.debug('Found Copyright text:' + str(element.text))
use_next = True
- if not copyright:
+ if not copyright_text:
for element in soup.findAll(['b', 'a'], text=True):
# LOG.debug("TEXT: "+element.text)
@@ -116,7 +115,7 @@ def _copyright(soup):
# which follows
sibling = element.next_sibling
stuff = ""
- while (sibling):
+ while sibling:
try:
stuff = stuff + sibling.text
except Exception:
@@ -124,17 +123,14 @@ def _copyright(soup):
sibling = sibling.next_sibling
if stuff:
- copyright = stuff.strip(' ')
+ copyright_text = stuff.strip(' ')
- return copyright
+ return copyright_text
except Exception as ex:
LOG.error(str(ex))
raise ValueError('Unsupported schema for given date.')
- # NO stated copyright, so we return None
- return None
-
def _explanation(soup):
"""
@@ -154,7 +150,7 @@ def _explanation(soup):
texts = [x.strip() for x in soup.text.split('\n')]
begin_idx = texts.index('Explanation:') + 1
idx = texts[begin_idx:].index('')
- s = (' ').join(texts[begin_idx:begin_idx + idx])
+ s = ' '.join(texts[begin_idx:begin_idx + idx])
return s
From 128cfc156b6ff70279c4ac66b0805b599fac9ec0 Mon Sep 17 00:00:00 2001
From: Jennifer Betancourt
Date: Wed, 12 Jul 2017 19:42:01 -0700
Subject: [PATCH 029/146] Changed python runtime version from 3.5.0 to 3.6.1
---
runtime.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/runtime.txt b/runtime.txt
index 294a23e..c91e43b 100644
--- a/runtime.txt
+++ b/runtime.txt
@@ -1 +1 @@
-python-3.5.0
+python-3.6.1
From 109173b4adfad2ad36613bf228f315ed92727d26 Mon Sep 17 00:00:00 2001
From: Jennifer Betancourt
Date: Wed, 12 Jul 2017 20:09:09 -0700
Subject: [PATCH 030/146] Fixed bug that occurs when server is a day ahead of
APOD website
---
apod/service.py | 9 +++++----
apod/utility.py | 1 +
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/apod/service.py b/apod/service.py
index 457d88d..06cfe1c 100644
--- a/apod/service.py
+++ b/apod/service.py
@@ -135,7 +135,6 @@ def _get_json_for_date(input_date, use_concept_tags):
# get data
data = _apod_handler(dt, use_concept_tags, use_default_today_date)
- data['date'] = input_date
data['service_version'] = SERVICE_VERSION
# return info as JSON
@@ -165,7 +164,6 @@ def _get_json_for_random_dates(count, use_concept_tags):
dt = date.fromordinal(date_ordinal)
data = _apod_handler(datetime.combine(dt, datetime.min.time()), use_concept_tags,
date_ordinal == today_ordinal)
- data['date'] = dt.isoformat()
data['service_version'] = SERVICE_VERSION
all_data.append(data)
@@ -208,9 +206,12 @@ def _get_json_for_date_range(start_date, end_date, use_concept_tags):
dt = date.fromordinal(start_ordinal)
data = _apod_handler(datetime.combine(dt, datetime.min.time()), use_concept_tags,
start_ordinal == today_ordinal)
- data['date'] = dt.isoformat()
data['service_version'] = SERVICE_VERSION
- all_data.append(data)
+
+ if data['date'] == dt.isoformat():
+ # Handles edge case where server is a day ahead of NASA APOD service
+ all_data.append(data)
+
start_ordinal += 1
# return info as JSON
diff --git a/apod/utility.py b/apod/utility.py
index bc4a5cf..787f818 100644
--- a/apod/utility.py
+++ b/apod/utility.py
@@ -52,6 +52,7 @@ def _get_apod_chars(dt):
props['copyright'] = copyright_text
props['media_type'] = media_type
props['url'] = data
+ props['date'] = dt.isoformat()
if hd_data:
props['hdurl'] = hd_data
From 12a479df1109598b94c7fd262f1af46838140ed9 Mon Sep 17 00:00:00 2001
From: Jennifer Betancourt
Date: Wed, 12 Jul 2017 20:11:26 -0700
Subject: [PATCH 031/146] Convert datetime object to date before getting
isoformat so that it doesn't include the time as well
---
apod/utility.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/apod/utility.py b/apod/utility.py
index 787f818..86cc843 100644
--- a/apod/utility.py
+++ b/apod/utility.py
@@ -52,7 +52,7 @@ def _get_apod_chars(dt):
props['copyright'] = copyright_text
props['media_type'] = media_type
props['url'] = data
- props['date'] = dt.isoformat()
+ props['date'] = dt.date().isoformat()
if hd_data:
props['hdurl'] = hd_data
From ba88a33cc6b30b974b449e8b393de57e76e63bf0 Mon Sep 17 00:00:00 2001
From: Jennifer Betancourt
Date: Wed, 12 Jul 2017 20:32:35 -0700
Subject: [PATCH 032/146] Updated README to include new API calls.
---
README.md | 3 +++
1 file changed, 3 insertions(+)
diff --git a/README.md b/README.md
index 19b33d5..4d02b70 100644
--- a/README.md
+++ b/README.md
@@ -14,6 +14,9 @@ as parameters to a http GET request. A JSON dictionary is returned nominally.
- `date` A string in YYYY-MM-DD format indicating the date of the APOD image (example: 2014-11-03). Defaults to today's date. Must be after 1995-06-16, the first day an APOD picture was posted. There are no images for tomorrow available through this API.
- `concept_tags` A boolean indicating whether concept tags should be returned with the rest of the response. The concept tags are not necessarily included in the explanation, but rather derived from common search tags that are associated with the description text. (Better than just pure text search.) Defaults to False.
- `hd` A boolean parameter indicating whether or not high-resolution images should be returned. This is present for legacy purposes, it is always ignored by the service and high-resolution urls are returned regardless.
+- `count` A positive integer, no greater than 100. If this is specified then `count` randomly chosen images will be returned in a JSON array. Cannot be used in conjunction with `date` or `start_date` and `end_date`.
+- `start_date` A string in YYYY-MM-DD format indicating the start of a date range. All images in the range from `start_date` to `end_date` will be returned in a JSON array. Cannot be used with `date`.
+- `end_date` A string in YYYY-MM-DD format indicating that end of a date range. If `start_date` is specified without an `end_date` then `end_date` defaults to the current date.
**Returned fields**
From 717fd1e8c72e0e96461cda3612591e6ea33a7c87 Mon Sep 17 00:00:00 2001
From: Jennifer Betancourt
Date: Mon, 17 Jul 2017 21:38:26 -0700
Subject: [PATCH 033/146] Updated README to include examples of API calls.
---
README.md | 94 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 94 insertions(+)
diff --git a/README.md b/README.md
index 4d02b70..874aa0f 100644
--- a/README.md
+++ b/README.md
@@ -69,6 +69,100 @@ localhost:5000/v1/apod?date=2014-10-01&concept_tags=True
}
```
+```bash
+https://api.nasa.gov/planetary/apod?api_key=DEMO_KEY&count=5
+```
+
+```jsoniq
+[
+ {
+ "copyright": "Panther Observatory",
+ "date": "2006-04-15",
+ "explanation": "In this stunning cosmic vista, galaxy M81 is on the left surrounded by blue spiral arms. On the right marked by massive gas and dust clouds, is M82. These two mammoth galaxies have been locked in gravitational combat for the past billion years. The gravity from each galaxy dramatically affects the other during each hundred million-year pass. Last go-round, M82's gravity likely raised density waves rippling around M81, resulting in the richness of M81's spiral arms. But M81 left M82 with violent star forming regions and colliding gas clouds so energetic the galaxy glows in X-rays. In a few billion years only one galaxy will remain.",
+ "hdurl": "https://apod.nasa.gov/apod/image/0604/M81_M82_schedler_c80.jpg",
+ "media_type": "image",
+ "service_version": "v1",
+ "title": "Galaxy Wars: M81 versus M82",
+ "url": "https://apod.nasa.gov/apod/image/0604/M81_M82_schedler_c25.jpg"
+ },
+ {
+ "date": "2013-07-22",
+ "explanation": "You are here. Everyone you've ever known is here. Every human who has ever lived -- is here. Pictured above is the Earth-Moon system as captured by the Cassini mission orbiting Saturn in the outer Solar System. Earth is the brighter and bluer of the two spots near the center, while the Moon is visible to its lower right. Images of Earth from Saturn were taken on Friday. Quickly released unprocessed images were released Saturday showing several streaks that are not stars but rather cosmic rays that struck the digital camera while it was taking the image. The above processed image was released earlier today. At nearly the same time, many humans on Earth were snapping their own pictures of Saturn. Note: Today's APOD has been updated.",
+ "hdurl": "https://apod.nasa.gov/apod/image/1307/earthmoon2_cassini_946.jpg",
+ "media_type": "image",
+ "service_version": "v1",
+ "title": "Earth and Moon from Saturn",
+ "url": "https://apod.nasa.gov/apod/image/1307/earthmoon2_cassini_960.jpg"
+ },
+ {
+ "copyright": "Joe Orman",
+ "date": "2000-04-06",
+ "explanation": "Rising before the Sun on February 2nd, astrophotographer Joe Orman anticipated this apparition of the bright morning star Venus near a lovely crescent Moon above a neighbor's house in suburban Phoenix, Arizona, USA. Fortunately, the alignment of bright planets and the Moon is one of the most inspiring sights in the night sky and one that is often easy to enjoy and share without any special equipment. Take tonight, for example. Those blessed with clear skies can simply step outside near sunset and view a young crescent Moon very near three bright planets in the west Jupiter, Mars, and Saturn. Jupiter will be the unmistakable brightest star near the Moon with a reddish Mars just to Jupiter's north and pale yellow Saturn directly above. Of course, these sky shows create an evocative picture but the planets and Moon just appear to be near each other -- they are actually only approximately lined up and lie in widely separated orbits. Unfortunately, next month's highly publicized alignment of planets on May 5th will be lost from view in the Sun's glare but such planetary alignments occur repeatedly and pose no danger to planet Earth.",
+ "hdurl": "https://apod.nasa.gov/apod/image/0004/vm_orman_big.jpg",
+ "media_type": "image",
+ "service_version": "v1",
+ "title": "Venus, Moon, and Neighbors",
+ "url": "https://apod.nasa.gov/apod/image/0004/vm_orman.jpg"
+ },
+ {
+ "date": "2014-07-12",
+ "explanation": "A new star, likely the brightest supernova in recorded human history, lit up planet Earth's sky in the year 1006 AD. The expanding debris cloud from the stellar explosion, found in the southerly constellation of Lupus, still puts on a cosmic light show across the electromagnetic spectrum. In fact, this composite view includes X-ray data in blue from the Chandra Observatory, optical data in yellowish hues, and radio image data in red. Now known as the SN 1006 supernova remnant, the debris cloud appears to be about 60 light-years across and is understood to represent the remains of a white dwarf star. Part of a binary star system, the compact white dwarf gradually captured material from its companion star. The buildup in mass finally triggered a thermonuclear explosion that destroyed the dwarf star. Because the distance to the supernova remnant is about 7,000 light-years, that explosion actually happened 7,000 years before the light reached Earth in 1006. Shockwaves in the remnant accelerate particles to extreme energies and are thought to be a source of the mysterious cosmic rays.",
+ "hdurl": "https://apod.nasa.gov/apod/image/1407/sn1006c.jpg",
+ "media_type": "image",
+ "service_version": "v1",
+ "title": "SN 1006 Supernova Remnant",
+ "url": "https://apod.nasa.gov/apod/image/1407/sn1006c_c800.jpg"
+ },
+ {
+ "date": "1997-01-21",
+ "explanation": "In Jules Verne's science fiction classic A Journey to the Center of the Earth, Professor Hardwigg and his fellow explorers encounter many strange and exciting wonders. What wonders lie at the center of our Galaxy? Astronomers now know of some of the bizarre objects which exist there, like vast dust clouds,\r bright young stars, swirling rings of gas, and possibly even a large black hole. Much of the Galactic center region is shielded from our view in visible light by the intervening dust and gas. But it can be explored using other forms of electromagnetic radiation, like radio, infrared, X-rays, and gamma rays. This beautiful high resolution image of the Galactic center region in infrared light was made by the SPIRIT III telescope onboard the Midcourse Space Experiment. The center itself appears as a bright spot near the middle of the roughly 1x3 degree field of view, the plane of the Galaxy is vertical, and the north galactic pole is towards the right. The picture is in false color - starlight appears blue while dust is greenish grey, tending to red in the cooler areas.",
+ "hdurl": "https://apod.nasa.gov/apod/image/9701/galcen_msx_big.gif",
+ "media_type": "image",
+ "service_version": "v1",
+ "title": "Journey to the Center of the Galaxy \r\nCredit:",
+ "url": "https://apod.nasa.gov/apod/image/9701/galcen_msx.jpg"
+ }
+]
+```
+
+```bash
+https://api.nasa.gov/planetary/apod?api_key=DEMO_KEY&start_date=2017-07-08&end_date=2017-07-10
+```
+
+
+```jsoniq
+[
+ {
+ "copyright": "T. Rector",
+ "date": "2017-07-08",
+ "explanation": "Similar in size to large, bright spiral galaxies in our neighborhood, IC 342 is a mere 10 million light-years distant in the long-necked, northern constellation Camelopardalis. A sprawling island universe, IC 342 would otherwise be a prominent galaxy in our night sky, but it is hidden from clear view and only glimpsed through the veil of stars, gas and dust clouds along the plane of our own Milky Way galaxy. Even though IC 342's light is dimmed by intervening cosmic clouds, this sharp telescopic image traces the galaxy's own obscuring dust, blue star clusters, and glowing pink star forming regions along spiral arms that wind far from the galaxy's core. IC 342 may have undergone a recent burst of star formation activity and is close enough to have gravitationally influenced the evolution of the local group of galaxies and the Milky Way.",
+ "hdurl": "https://apod.nasa.gov/apod/image/1707/ic342_rector2048.jpg",
+ "media_type": "image",
+ "service_version": "v1",
+ "title": "Hidden Galaxy IC 342",
+ "url": "https://apod.nasa.gov/apod/image/1707/ic342_rector1024s.jpg"
+ },
+ {
+ "date": "2017-07-09",
+ "explanation": "Can you find your favorite country or city? Surprisingly, on this world-wide nightscape, city lights make this task quite possible. Human-made lights highlight particularly developed or populated areas of the Earth's surface, including the seaboards of Europe, the eastern United States, and Japan. Many large cities are located near rivers or oceans so that they can exchange goods cheaply by boat. Particularly dark areas include the central parts of South America, Africa, Asia, and Australia. The featured composite was created from images that were collected during cloud-free periods in April and October 2012 by the Suomi-NPP satellite, from a polar orbit about 824 kilometers above the surface, using its Visible Infrared Imaging Radiometer Suite (VIIRS).",
+ "hdurl": "https://apod.nasa.gov/apod/image/1707/EarthAtNight_SuomiNPP_3600.jpg",
+ "media_type": "image",
+ "service_version": "v1",
+ "title": "Earth at Night",
+ "url": "https://apod.nasa.gov/apod/image/1707/EarthAtNight_SuomiNPP_1080.jpg"
+ },
+ {
+ "date": "2017-07-10",
+ "explanation": "What's happening around the center of this spiral galaxy? Seen in total, NGC 1512 appears to be a barred spiral galaxy -- a type of spiral that has a straight bar of stars across its center. This bar crosses an outer ring, though, a ring not seen as it surrounds the pictured region. Featured in this Hubble Space Telescope image is an inner ring -- one that itself surrounds the nucleus of the spiral. The two rings are connected not only by a bar of bright stars but by dark lanes of dust. Inside of this inner ring, dust continues to spiral right into the very center -- possibly the location of a large black hole. The rings are bright with newly formed stars which may have been triggered by the collision of NGC 1512 with its galactic neighbor, NGC 1510.",
+ "hdurl": "https://apod.nasa.gov/apod/image/1707/NGC1512_Schmidt_1342.jpg",
+ "media_type": "image",
+ "service_version": "v1",
+ "title": "Spiral Galaxy NGC 1512: The Nuclear Ring",
+ "url": "https://apod.nasa.gov/apod/image/1707/NGC1512_Schmidt_960.jpg"
+ }
+]
+```
+
## Getting started
1. Install the [App Engine Python SDK](https://developers.google.com/appengine/downloads).
From 4fe5adad1f98796685c8be20dcf5c4a0cbb01c84 Mon Sep 17 00:00:00 2001
From: Jennifer Betancourt
Date: Thu, 20 Jul 2017 08:20:58 -0700
Subject: [PATCH 034/146] Fixed error when parsing some older dates.
---
apod/utility.py | 14 +++++++++++++-
1 file changed, 13 insertions(+), 1 deletion(-)
diff --git a/apod/utility.py b/apod/utility.py
index 86cc843..524ebad 100644
--- a/apod/utility.py
+++ b/apod/utility.py
@@ -149,7 +149,19 @@ def _explanation(soup):
if s == '':
# Handler for earlier APOD entries
texts = [x.strip() for x in soup.text.split('\n')]
- begin_idx = texts.index('Explanation:') + 1
+ try:
+ begin_idx = texts.index('Explanation:') + 1
+ except ValueError as e:
+ # Rare case where "Explanation:" is not on its own line
+ explanation_line = [x for x in texts if "Explanation:" in x]
+ if len(explanation_line) == 1:
+ begin_idx = texts.index(explanation_line[0])
+ texts[begin_idx] = texts[begin_idx][12:].strip()
+ else:
+ raise e
+
+
+
idx = texts[begin_idx:].index('')
s = ' '.join(texts[begin_idx:begin_idx + idx])
return s
From 91b67d636f1066f290ef5b96d24029d2f3603cca Mon Sep 17 00:00:00 2001
From: jnbetanc
Date: Mon, 25 Sep 2017 09:39:14 -0700
Subject: [PATCH 035/146] Switched datetime objects to date objects since the
time of day information was not relevant, and cleaned up some comments and
unused code
---
apod/service.py | 25 ++++---------------------
apod/utility.py | 11 ++---------
2 files changed, 6 insertions(+), 30 deletions(-)
diff --git a/apod/service.py b/apod/service.py
index 06cfe1c..3386387 100644
--- a/apod/service.py
+++ b/apod/service.py
@@ -22,7 +22,6 @@
LOG = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)
-# LOG.setLevel(logging.DEBUG)
# this should reflect both this service and the backing
# assorted libraries
@@ -62,20 +61,6 @@ def _validate(data):
return True
-# TODO(jbetancourt) Convert all datetime objects to dates, then remove this function
-def _validate_datetime(dt):
- LOG.debug('_validate_datetime(dt) called')
- today = datetime.today()
- begin = datetime(1995, 6, 16) # first APOD image date
-
- # validate input
- if (dt > today) or (dt < begin):
- today_str = today.strftime('%b %d, %Y')
- begin_str = begin.strftime('%b %d, %Y')
-
- raise ValueError('Date must be between %s and %s.' % (begin_str, today_str))
-
-
def _validate_date(dt):
LOG.debug('_validate_date(dt) called')
today = datetime.today().date()
@@ -130,8 +115,8 @@ def _get_json_for_date(input_date, use_concept_tags):
use_default_today_date = True
# validate input date
- dt = datetime.strptime(input_date, '%Y-%m-%d')
- _validate_datetime(dt)
+ dt = datetime.strptime(input_date, '%Y-%m-%d').date()
+ _validate_date(dt)
# get data
data = _apod_handler(dt, use_concept_tags, use_default_today_date)
@@ -162,8 +147,7 @@ def _get_json_for_random_dates(count, use_concept_tags):
all_data = []
for date_ordinal in random_date_ordinals:
dt = date.fromordinal(date_ordinal)
- data = _apod_handler(datetime.combine(dt, datetime.min.time()), use_concept_tags,
- date_ordinal == today_ordinal)
+ data = _apod_handler(dt, use_concept_tags, date_ordinal == today_ordinal)
data['service_version'] = SERVICE_VERSION
all_data.append(data)
@@ -204,8 +188,7 @@ def _get_json_for_date_range(start_date, end_date, use_concept_tags):
while start_ordinal <= end_ordinal:
# get data
dt = date.fromordinal(start_ordinal)
- data = _apod_handler(datetime.combine(dt, datetime.min.time()), use_concept_tags,
- start_ordinal == today_ordinal)
+ data = _apod_handler(dt, use_concept_tags, start_ordinal == today_ordinal)
data['service_version'] = SERVICE_VERSION
if data['date'] == dt.isoformat():
diff --git a/apod/utility.py b/apod/utility.py
index 524ebad..d582fe2 100644
--- a/apod/utility.py
+++ b/apod/utility.py
@@ -14,7 +14,6 @@
LOG = logging.getLogger(__name__)
logging.basicConfig(level=logging.WARN)
-# LOG.setLevel(logging.DEBUG)
# location of backing APOD service
BASE = 'https://apod.nasa.gov/apod/'
@@ -52,7 +51,7 @@ def _get_apod_chars(dt):
props['copyright'] = copyright_text
props['media_type'] = media_type
props['url'] = data
- props['date'] = dt.date().isoformat()
+ props['date'] = dt.isoformat()
if hd_data:
props['hdurl'] = hd_data
@@ -76,8 +75,6 @@ def _title(soup):
# Handler for early APOD entries
text = soup.title.text.split(' - ')[-1]
return text.strip()
- else:
- raise ValueError('Unsupported schema for given date.')
def _copyright(soup):
@@ -108,12 +105,10 @@ def _copyright(soup):
if not copyright_text:
for element in soup.findAll(['b', 'a'], text=True):
- # LOG.debug("TEXT: "+element.text)
# search text for explicit match
if 'Copyright' in element.text:
LOG.debug('Found Copyright text:' + str(element.text))
- # pull the copyright from the link text
- # which follows
+ # pull the copyright from the link text which follows
sibling = element.next_sibling
stuff = ""
while sibling:
@@ -160,8 +155,6 @@ def _explanation(soup):
else:
raise e
-
-
idx = texts[begin_idx:].index('')
s = ' '.join(texts[begin_idx:begin_idx + idx])
return s
From 918cf0e39242c469d631258b1a433a681cac32ba Mon Sep 17 00:00:00 2001
From: jnbetanc
Date: Tue, 31 Jul 2018 18:51:46 -0700
Subject: [PATCH 036/146] Upgrade gunicorn to 19.5.0
---
requirements.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/requirements.txt b/requirements.txt
index 5d9e82b..53941e1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,7 +5,7 @@
#
# Note: The `lib` directory is added to `sys.path` by `appengine_config.py`.
Flask-Cors==2.1.2
-gunicorn==19.3.0
+gunicorn==19.5.0
Jinja2==2.8
Werkzeug==0.10.4
beautifulsoup4==4.5.3
From cff911425b2c8e921b1d5f803a3af40fc68a3671 Mon Sep 17 00:00:00 2001
From: PawelPleskaczynski
Date: Wed, 10 Oct 2018 23:16:08 +0200
Subject: [PATCH 037/146] fixed 500 error that was thrown while requesting
particular dates
---
apod/utility.py | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
diff --git a/apod/utility.py b/apod/utility.py
index d582fe2..97933c3 100644
--- a/apod/utility.py
+++ b/apod/utility.py
@@ -4,6 +4,9 @@
Created on Mar 24, 2017
@author=bathomas @email=brian.a.thomas@nasa.gov
+
+Modified on Oct 10, 2018
+@author=PawelPleskaczynski @email=pawelpleskaczynski@gmail.com
"""
from bs4 import BeautifulSoup
@@ -37,10 +40,14 @@ def _get_apod_chars(dt):
if link['href'] and link['href'].startswith('image'):
hd_data = BASE + link['href']
break
- else:
+ elif soup.iframe:
# its a video
media_type = 'video'
data = soup.iframe['src']
+ else:
+ # it is neither image nor video, output empty urls
+ media_type = 'other'
+ data = ''
props = {}
@@ -50,7 +57,8 @@ def _get_apod_chars(dt):
if copyright_text:
props['copyright'] = copyright_text
props['media_type'] = media_type
- props['url'] = data
+ if data:
+ props['url'] = data
props['date'] = dt.isoformat()
if hd_data:
From bcc60913e4fcba31768dafc731034f338dce714f Mon Sep 17 00:00:00 2001
From: PawelPleskaczynski
Date: Sat, 13 Oct 2018 18:55:14 +0200
Subject: [PATCH 038/146] added a query parameter to output video thumbnails
---
README.md | 2 ++
apod/service.py | 27 ++++++++++++------------
apod/templates/home.html | 25 +++++++++++-----------
apod/utility.py | 45 ++++++++++++++++++++++++++++++++--------
4 files changed, 65 insertions(+), 34 deletions(-)
diff --git a/README.md b/README.md
index 874aa0f..a0856a5 100644
--- a/README.md
+++ b/README.md
@@ -17,6 +17,7 @@ as parameters to a http GET request. A JSON dictionary is returned nominally.
- `count` A positive integer, no greater than 100. If this is specified then `count` randomly chosen images will be returned in a JSON array. Cannot be used in conjunction with `date` or `start_date` and `end_date`.
- `start_date` A string in YYYY-MM-DD format indicating the start of a date range. All images in the range from `start_date` to `end_date` will be returned in a JSON array. Cannot be used with `date`.
- `end_date` A string in YYYY-MM-DD format indicating that end of a date range. If `start_date` is specified without an `end_date` then `end_date` defaults to the current date.
+- `thumbs` If set to `true`, the API returns URL of video thumbnail. If an APOD is not a video, this parameter is ignored.
**Returned fields**
@@ -29,6 +30,7 @@ as parameters to a http GET request. A JSON dictionary is returned nominally.
- `media_type` The type of media (data) returned. May either be 'image' or 'video' depending on content.
- `explanation` The supplied text explanation of the image.
- `concepts` The most relevant concepts within the text explanation. Only supplied if `concept_tags` is set to True.
+- `thumbnail_url` The URL of thumbnail of the video.
**Example**
diff --git a/apod/service.py b/apod/service.py
index 3386387..79e4971 100644
--- a/apod/service.py
+++ b/apod/service.py
@@ -27,7 +27,7 @@
# assorted libraries
SERVICE_VERSION = 'v1'
APOD_METHOD_NAME = 'apod'
-ALLOWED_APOD_FIELDS = ['concept_tags', 'date', 'hd', 'count', 'start_date', 'end_date']
+ALLOWED_APOD_FIELDS = ['concept_tags', 'date', 'hd', 'count', 'start_date', 'end_date', 'thumbs']
ALCHEMY_API_KEY = None
try:
@@ -74,13 +74,13 @@ def _validate_date(dt):
raise ValueError('Date must be between %s and %s.' % (begin_str, today_str))
-def _apod_handler(dt, use_concept_tags=False, use_default_today_date=False):
+def _apod_handler(dt, use_concept_tags=False, use_default_today_date=False, thumbs=False):
"""
Accepts a parameter dictionary. Returns the response object to be
served through the API.
"""
try:
- page_props = parse_apod(dt, use_default_today_date)
+ page_props = parse_apod(dt, use_default_today_date, thumbs)
LOG.debug('managed to get apod page characteristics')
if use_concept_tags:
@@ -98,7 +98,7 @@ def _apod_handler(dt, use_concept_tags=False, use_default_today_date=False):
return _abort(500, 'Internal Service Error', usage=False)
-def _get_json_for_date(input_date, use_concept_tags):
+def _get_json_for_date(input_date, use_concept_tags, thumbs):
"""
This returns the JSON data for a specific date, which must be a string of the form YYYY-MM-DD. If date is None,
then it defaults to the current date.
@@ -119,14 +119,14 @@ def _get_json_for_date(input_date, use_concept_tags):
_validate_date(dt)
# get data
- data = _apod_handler(dt, use_concept_tags, use_default_today_date)
+ data = _apod_handler(dt, use_concept_tags, use_default_today_date, thumbs)
data['service_version'] = SERVICE_VERSION
# return info as JSON
return jsonify(data)
-def _get_json_for_random_dates(count, use_concept_tags):
+def _get_json_for_random_dates(count, use_concept_tags, thumbs):
"""
This returns the JSON data for a set of randomly chosen dates. The number of dates is specified by the count
parameter
@@ -147,14 +147,14 @@ def _get_json_for_random_dates(count, use_concept_tags):
all_data = []
for date_ordinal in random_date_ordinals:
dt = date.fromordinal(date_ordinal)
- data = _apod_handler(dt, use_concept_tags, date_ordinal == today_ordinal)
+ data = _apod_handler(dt, use_concept_tags, date_ordinal == today_ordinal, thumbs)
data['service_version'] = SERVICE_VERSION
all_data.append(data)
return jsonify(all_data)
-def _get_json_for_date_range(start_date, end_date, use_concept_tags):
+def _get_json_for_date_range(start_date, end_date, use_concept_tags, thumbs):
"""
This returns the JSON data for a range of dates, specified by start_date and end_date, which must be strings of the
form YYYY-MM-DD. If end_date is None then it defaults to the current date.
@@ -188,7 +188,7 @@ def _get_json_for_date_range(start_date, end_date, use_concept_tags):
while start_ordinal <= end_ordinal:
# get data
dt = date.fromordinal(start_ordinal)
- data = _apod_handler(dt, use_concept_tags, start_ordinal == today_ordinal)
+ data = _apod_handler(dt, use_concept_tags, start_ordinal == today_ordinal, thumbs)
data['service_version'] = SERVICE_VERSION
if data['date'] == dt.isoformat():
@@ -218,7 +218,7 @@ def apod():
LOG.info('apod path called')
try:
- # application/json GET method
+ # application/json GET method
args = request.args
if not _validate(args):
@@ -230,15 +230,16 @@ def apod():
start_date = args.get('start_date')
end_date = args.get('end_date')
use_concept_tags = args.get('concept_tags', False)
+ thumbs = args.get('thumbs', False)
if not count and not start_date and not end_date:
- return _get_json_for_date(input_date, use_concept_tags)
+ return _get_json_for_date(input_date, use_concept_tags, thumbs)
elif not input_date and not start_date and not end_date and count:
- return _get_json_for_random_dates(int(count), use_concept_tags)
+ return _get_json_for_random_dates(int(count), use_concept_tags, thumbs)
elif not count and not input_date and start_date:
- return _get_json_for_date_range(start_date, end_date, use_concept_tags)
+ return _get_json_for_date_range(start_date, end_date, use_concept_tags, thumbs)
else:
return _abort(400, 'Bad Request: invalid field combination passed.')
diff --git a/apod/templates/home.html b/apod/templates/home.html
index 86ac35f..c602b05 100644
--- a/apod/templates/home.html
+++ b/apod/templates/home.html
@@ -9,7 +9,7 @@ NASA/OCIO Astronomy Picture Of the Day (APOD) Service
Service API
-This service contains a single endpoint, "/{{ version }}/{{ methodname }}/",
+This service contains a single endpoint, "/{{ version }}/{{ methodname }}/",
which may be used to obtain a selected image url and metadata from http://apod.nasa.gov.
You can use this service endpoint by sending a GET request which may contain one or
@@ -17,12 +17,12 @@
Service API
| Allowed Field | Description |
-| date | A string in YYYY-MM-DD format indicating the date of the APOD image
-(example: 2014-11-03). Must be after 1995-06-16, the first day an APOD picture was posted.
-There are no images for tomorrow available through this API. Defaults to today's date. |
-| concept_tags | A boolean indicating whether concept tags should be returned with the
-rest of the response. The concept tags are not necessarily included in the explanation, but
-rather derived from common search tags that are associated with the description text.
+ |
| date | A string in YYYY-MM-DD format indicating the date of the APOD image
+(example: 2014-11-03). Must be after 1995-06-16, the first day an APOD picture was posted.
+There are no images for tomorrow available through this API. Defaults to today's date. |
+| concept_tags | A boolean indicating whether concept tags should be returned with the
+rest of the response. The concept tags are not necessarily included in the explanation, but
+rather derived from common search tags that are associated with the description text.
(Better than just pure text search.). Defaults to False. |
@@ -34,8 +34,8 @@
Service API
-which should return an application/json response with a JSON formatted string containing
-the desired information.
+which should return an application/json response with a JSON formatted string containing
+the desired information.
For example, the return JSON from the above query is:
@@ -53,13 +53,14 @@ Service API
| Returned Field | Description |
-| resource | A dictionary describing the `image_set` or `planet` that the
+ |
| resource | A dictionary describing the `image_set` or `planet` that the
response illustrates, completely determined by the structured endpoint |
| concept_tags | A boolean reflection of the supplied option. Included in response because of default values. |
-| title | The title of the image. |
+| title | The title of the image. |
| date | Date of image. Included in response because of default values. |
-| url | The URL of the APOD image of the day. |
+| url | The URL of the APOD image of the day. |
| explanation | The supplied text explanation of the image. |
| concepts | The most relevant concepts within the text explanation. Only supplied if `concept_tags` is set to True. |
+| thumbnail_url | The URL of thumbnail of the video. Only supplied if `thumbs` is set to True. |