import json
import os
import os.path
from urllib import request
import requests
from bs4 import BeautifulSoup
from pyavm import AVM
from pyavm.avm import AVMContainer
def convert_avm_container(s):
avm_items = s._items
return {
k : convert_avm_container(avm_items.get(k)) if isinstance(avm_items.get(k), AVMContainer) else avm_items.get(k)
for k in avm_items.keys()
if avm_items.get(k) is not None
}
# res = {}
# for k in dir(s):
# v = getattr(s,k)
# if v is None:
# continue
# if isinstance(v, AVMContainer):
# print("%s IS AVMContainer %r" % (k, v))
# res[k] = convert_avm_container(v)
# else:
# res[k] = v
# return res
for page in range (1, 10):
url = "http://www.spitzer.caltech.edu/search/image_set/100?resource=image_set&tabs=hidden&page={}".format(page)
website = requests.get(url)
soup = BeautifulSoup(website.content, "html5lib")
table = soup.find("table", {"class":"items"})
itemtds = table.find_all("td", {"class":"item"})
if len(itemtds) == 0:
print("No item in page, exit.")
break
for td in itemtds:
# itemlinks = find("a", href=True)
# print(td.div.a.get('href'))
detail_url = "http://www.spitzer.caltech.edu" + td.div.a.get('href')
detail_content = requests.get(detail_url).content
detail_soup = BeautifulSoup(detail_content, "html5lib")
img_id_elem = detail_soup.find("dd", {"property":"avm:ID"})
img_id = img_id_elem.getText()
img_url = None
img_box = detail_soup.find("div", {"class":"sidebar-section download"})
for img_link in img_box.find_all("a"):
img_link_href = img_link.get("href")
if img_link_href.endswith(img_id + ".jpg"):
img_url = "http://www.spitzer.caltech.edu" + img_link_href
break
print(img_url)
if not img_url:
continue
if not os.path.isdir('scrapSpitzer/' + img_id):
os.makedirs('scrapSpitzer/' + img_id, exist_ok=True)
img_path = 'scrapSpitzer/{0}/{0}.jpg'.format(img_id)
json_path = 'scrapSpitzer/{0}/{0}.json'.format(img_id)
if os.path.isfile(img_path):
print("--> file %s exists from url %s" % (img_path,img_url))
continue
request.urlretrieve(img_url, img_path)
avm = AVM.from_image(img_path)
img_data = convert_avm_container(avm)
img_json = {
'avm': img_data,
'image': {
'id': img_data.get('ID'),
'title': img_data.get('Title'),
'description_text': img_data.get('Description'),
'description_html': '<div>' + img_data.get('Description') + "</div>",
'date_publication' : img_data.get('Date'),
'credit': img_data.get('Credit') or 'Courtesy NASA/JPL-Caltech',
'type': img_data.get('Type'),
'imgurl': img_data.get('ResourceURL')
},
'url': img_data.get('ReferenceURL')
}
with open(json_path, 'w') as outfile:
json.dump(img_json, outfile)
'''
MetadataVersion: b'1.2'
Creator: b'Spitzer Space Telescope'
Title: b'Surface of TRAPPIST-1f'
Headline: b"Imagine standing on the surface of the exoplanet TRAPPIST-1f. This artist's concept is one interpretation of what it could look like. "
Description: b"This artist's concept allows us to imagine what it would be like to stand on the surface of the exoplanet TRAPPIST-1f, located in the TRAPPIST-1 system in the constellation Aquarius. \n\nBecause this planet is thought to be tidally locked to its star, meaning the same face of the planet is always pointed at the star, there would be a region called the terminator that perpetually divides day and night. If the night side is icy, the day side might give way to liquid water in the area where sufficient starlight hits the surface. \n\nOne of the unusual features of TRAPPIST-1 planets is how close they are to each other -- so close that other planets could be visible in the sky from the surface of each one. In this view, the planets in the sky correspond to TRAPPIST1e (top left crescent), d (middle crescent) and c (bright dot to the lower right of the crescents). TRAPPIST-1e would appear about the same size as the moon and TRAPPIST1-c is on the far side of the star. The star itself, an ultra-cool dwarf, would appear about three times larger than our own sun does in Earth's skies.\n\nThe TRAPPIST-1 system has been revealed through observations from NASA's Spitzer Space Telescope and the ground-based TRAPPIST (TRAnsiting Planets and PlanetesImals Small Telescope) telescope, as well as other ground-based observatories. The system was named for the TRAPPIST telescope.\n\nNASA's Jet Propulsion Laboratory, Pasadena, California, manages the Spitzer Space Telescope mission for NASA's Science Mission Directorate, Washington. Science operations are conducted at the Spitzer Science Center at Caltech in Pasadena. Spacecraft operations are based at Lockheed Martin Space Systems Company, Littleton, Colorado. Data are archived at the Infrared Science Archive housed at Caltech/IPAC. Caltech manages JPL for NASA. "
ReferenceURL: b'http://www.spitzer.caltech.edu/images/6274-ssc2017-01c-Surface-of-TRAPPIST-1f'
Credit: b'NASA/JPL-Caltech/T. Pyle (IPAC)'
Date: b'2017-02-22'
ID: b'ssc2017-01c'
Type: b'Artwork'
Publisher: b'Spitzer Science Center'
PublisherID: b'spitzer'
ResourceID: b'ssc2017-01c.jpg'
ResourceURL: b'http://www.spitzer.caltech.edu/uploaded_files/images/0010/9932/ssc201
'''