import json

import os

from urllib import request

import requests

from bs4 import BeautifulSoup

from pyavm import AVM

from pyavm.avm import AVMContainer

def convert_avm_container(s):

    avm_items = s._items

    return {

        k : convert_avm_container(avm_items.get(k)) if isinstance(avm_items.get(k), AVMContainer) else avm_items.get(k)

        for k in avm_items.keys()

        if avm_items.get(k) is not None

    }

    # res = {}

    # for k in dir(s):

    #     v = getattr(s,k)

    #     if v is None:

    #         continue

    #     if isinstance(v, AVMContainer):

    #         print("%s IS AVMContainer %r" % (k, v))

    #         res[k] = convert_avm_container(v)

    #     else:

    #         res[k] = v

    # return res

for page in range (1, 10):

    website = requests.get(url)

    soup = BeautifulSoup(website.content, "html5lib")

    table = soup.find("table", {"class":"items"})

    itemtds = table.find_all("td", {"class":"item"})

    for td in itemtds:

    # itemlinks = find("a", href=True)

        # print(td.div.a.get('href'))

        detail_url = "http://www.spitzer.caltech.edu" + td.div.a.get('href')

        detail_content = requests.get(detail_url).content

        detail_soup = BeautifulSoup(detail_content, "html5lib")

        img_id_elem = detail_soup.find("dd", {"property":"avm:ID"})

        img_id = img_id_elem.getText()

        img_url = None

        img_box = detail_soup.find("div", {"class":"sidebar-section download"})

        for img_link in img_box.find_all("a"):

            img_link_href = img_link.get("href")

            if img_link_href.endswith(img_id + ".jpg"):

                img_url = "http://www.spitzer.caltech.edu" + img_link_href

                break

        print(img_url)

        if not img_url:

            continue

        if not os.path.isdir('scrapSpitzer/' + img_id):

            os.makedirs('scrapSpitzer/' + img_id, exist_ok=True)

        img_path = 'scrapSpitzer/{0}/{0}.jpg'.format(img_id)

        json_path = 'scrapSpitzer/{0}/{0}.json'.format(img_id)

        request.urlretrieve(img_url, img_path)

        avm = AVM.from_image(img_path)

        img_data = convert_avm_container(avm)

        img_json = { 

            'avm': img_data,

            'image': {

                'id': img_data.get('ID'),

                'title': img_data.get('Title'),

                'description_text': img_data.get('Description'),

                'description_html': '<div>' + img_data.get('Description') + "</div>",

                'date_publication' : img_data.get('Date'),

                'credit': img_data.get('Credit') or 'Courtesy NASA/JPL-Caltech',

                'type': img_data.get('Type'),

                'imgurl': img_data.get('ResourceURL')

            },

            'url': img_data.get('ReferenceURL')

        }

        with open(json_path, 'w') as outfile:

            json.dump(img_json, outfile)

'''

MetadataVersion: b'1.2'

Creator: b'Spitzer Space Telescope'

Title: b'Surface of TRAPPIST-1f'

Headline: b"Imagine standing on the surface of the exoplanet TRAPPIST-1f. This artist's concept is one interpretation of what it could look like. "

Description: b"This artist's concept allows us to imagine what it would be like to stand on the surface of the exoplanet TRAPPIST-1f, located in the TRAPPIST-1 system in the constellation Aquarius. \n\nBecause this planet is thought to be tidally locked to its star, meaning the same face of the planet is always pointed at the star, there would be a region called the terminator that perpetually divides day and night. If the night side is icy, the day side might give way to liquid water in the area where sufficient starlight hits the surface. \n\nOne of the unusual features of TRAPPIST-1 planets is how close they are to each other -- so close that other planets could be visible in the sky from the surface of each one. In this view, the planets in the sky correspond to TRAPPIST1e (top left crescent), d (middle crescent) and c (bright dot to the lower right of the crescents). TRAPPIST-1e would appear about the same size as the moon and TRAPPIST1-c is on the far side of the star. The star itself, an ultra-cool dwarf, would appear about three times larger than our own sun does in Earth's skies.\n\nThe TRAPPIST-1 system has been revealed through observations from NASA's Spitzer Space Telescope and the ground-based TRAPPIST (TRAnsiting Planets and PlanetesImals Small Telescope) telescope, as well as other ground-based observatories. The system was named for the TRAPPIST telescope.\n\nNASA's Jet Propulsion Laboratory, Pasadena, California, manages the Spitzer Space Telescope mission for NASA's Science Mission Directorate, Washington. Science operations are conducted at the Spitzer Science Center at Caltech in Pasadena. Spacecraft operations are based at Lockheed Martin Space Systems Company, Littleton, Colorado. Data are archived at the Infrared Science Archive housed at Caltech/IPAC. Caltech manages JPL for NASA. "

ReferenceURL: b'http://www.spitzer.caltech.edu/images/6274-ssc2017-01c-Surface-of-TRAPPIST-1f'

Credit: b'NASA/JPL-Caltech/T. Pyle (IPAC)'

Date: b'2017-02-22'

ID: b'ssc2017-01c'

Type: b'Artwork'

Publisher: b'Spitzer Science Center'

PublisherID: b'spitzer'

ResourceID: b'ssc2017-01c.jpg'

ResourceURL: b'http://www.spitzer.caltech.edu/uploaded_files/images/0010/9932/ssc201

'''