import glob
import json
import os
import os.path
import re
import shutil
import sys
import traceback
from urllib import parse as urlparse
from urllib import request

import requests

from bs4 import BeautifulSoup
from PIL import Image
from pyavm import AVM
from pyavm.avm import AVMContainer


def convert_avm_container(s):
    avm_items = s._items
    return {
        k : convert_avm_container(avm_items.get(k)) if isinstance(avm_items.get(k), AVMContainer) else avm_items.get(k)
        for k in avm_items.keys()
        if avm_items.get(k) is not None
    }
    # res = {}
    # for k in dir(s):
    #     v = getattr(s,k)
    #     if v is None:
    #         continue
    #     if isinstance(v, AVMContainer):
    #         print("%s IS AVMContainer %r" % (k, v))
    #         res[k] = convert_avm_container(v)
    #     else:
    #         res[k] = v
    # return res
IMG_LINK_RE = re.compile(r"(\d+)\s*x\s*(\d+)\s")

def parse_img_link_size(img_link_txt):
    m = IMG_LINK_RE.search(img_link_txt)
    return int(m.group(1))*int(m.group(2)) if m else 0

for page in range (1, 10):
    url = "http://www.spitzer.caltech.edu/search/image_set/100?resource=image_set&tabs=hidden&page={}".format(page)
    website = requests.get(url)
    soup = BeautifulSoup(website.content, "html5lib")

    table = soup.find("table", {"class":"items"})
    itemtds = table.find_all("td", {"class":"item"})

    if len(itemtds) == 0:
        print("No item in page, exit.")
        break

    for td in itemtds:
    # itemlinks = find("a", href=True)
        img_href = td.div.a.get('href')
        print("IMG HREF %s" % img_href)
        img_slug = img_href.split("/")[-1]
        img_dir_path = 'scrapSpitzer/' + img_slug
        if os.path.isdir(img_dir_path) and glob.glob("%s/*.jpg"%img_dir_path) and glob.glob("%s/*.json"%img_dir_path):
            print("--> img %s exists" % (img_slug,))
            continue
        else:
            os.makedirs(img_dir_path, exist_ok=True)

        detail_url = "http://www.spitzer.caltech.edu" + img_href

        try:
            detail_content = requests.get(detail_url).content
            detail_soup = BeautifulSoup(detail_content, "html5lib")
            img_id_elem = detail_soup.find("dd", {"property":"avm:ID"})
            img_id = img_id_elem.getText().strip() if img_id_elem else None
            if not img_id:
                print("--> img %s has no id, skipping" % img_href)
                continue
            img_url = None
            img_box = detail_soup.find("div", {"class":"sidebar-section download"})

            img_link_href_array = [ 
                (img_link.get("href"), parse_img_link_size(img_link.getText()))
                for img_link in img_box.find_all("a") ]

            for img_link_href in img_link_href_array:
                if img_link_href[0].endswith(img_id + ".jpg"):
                    img_url = "http://www.spitzer.caltech.edu" + img_link_href[0]
                    break

            if not img_url:
                jpeg_img_link_href_array = sorted(filter(lambda l: l[0].endswith(".jpg"), img_link_href_array), key=lambda e: e[1], reverse=True)
                if jpeg_img_link_href_array:
                    img_url = "http://www.spitzer.caltech.edu" + jpeg_img_link_href_array[0][0]


            if not img_url:
                tiff_img_link_href_array = sorted(filter(lambda l: l[0].endswith(".tif"), img_link_href_array), key=lambda e: e[1], reverse=True)
                if tiff_img_link_href_array:
                    img_url = "http://www.spitzer.caltech.edu" + tiff_img_link_href_array[0][0]


            if not img_url:
                print("NO IMG URL for %s : %r" % (img_id, img_link_href_array))
                continue
            print("IMG URL %s" % img_url)
            p = urlparse.urlparse(img_url).path
            img_ext = os.path.splitext(p)[1]
            orig_img_path = 'scrapSpitzer/{0}/{1}{2}'.format(img_slug, img_id, img_ext)
            img_path = 'scrapSpitzer/{0}/{1}.jpg'.format(img_slug, img_id)
            json_path = 'scrapSpitzer/{0}/{1}.json'.format(img_slug, img_id)
            if os.path.isfile(img_path) and os.path.isfile(json_path):
                print("--> file %s exists from url %s" % (img_path,img_url))
                continue
            
            if not os.path.isfile(orig_img_path):
                request.urlretrieve(img_url, orig_img_path)

            if img_ext != "jpg" and not os.path.isfile(img_path) :
                im = Image.open(orig_img_path)
                rgb_im = im.convert('RGB')
                rgb_im.save(img_path)

            avm = AVM.from_image(orig_img_path)
            img_data = convert_avm_container(avm)
            description = img_data.get('Description') or ""

            img_json = { 
                'avm': img_data,
                'image': {
                    'id': img_data.get('ID'),
                    'title': img_data.get('Title'),
                    'description_text': description,
                    'description_html': '<div>' + description + "</div>",
                    'date_publication' : img_data.get('Date'),
                    'credit': img_data.get('Credit') or 'Courtesy NASA/JPL-Caltech',
                    'type': img_data.get('Type'),
                    'imgurl': img_data.get('ResourceURL')
                },
                'url': img_data.get('ReferenceURL')
            }

            with open(json_path, 'w') as outfile:
                json.dump(img_json, outfile)

            if img_ext != "jpg":
                avm.embed(img_path, img_path)

        except (KeyboardInterrupt, SystemExit):
            raise
        except:
            print("--> error processing %s" % (img_href,))
            traceback.print_exc(file=sys.stdout)

# '''

# MetadataVersion: b'1.2'
# Creator: b'Spitzer Space Telescope'
# Title: b'Surface of TRAPPIST-1f'
# Headline: b"Imagine standing on the surface of the exoplanet TRAPPIST-1f. This artist's concept is one interpretation of what it could look like. "
# Description: b"This artist's concept allows us to imagine what it would be like to stand on the surface of the exoplanet TRAPPIST-1f, located in the TRAPPIST-1 system in the constellation Aquarius. \n\nBecause this planet is thought to be tidally locked to its star, meaning the same face of the planet is always pointed at the star, there would be a region called the terminator that perpetually divides day and night. If the night side is icy, the day side might give way to liquid water in the area where sufficient starlight hits the surface. \n\nOne of the unusual features of TRAPPIST-1 planets is how close they are to each other -- so close that other planets could be visible in the sky from the surface of each one. In this view, the planets in the sky correspond to TRAPPIST1e (top left crescent), d (middle crescent) and c (bright dot to the lower right of the crescents). TRAPPIST-1e would appear about the same size as the moon and TRAPPIST1-c is on the far side of the star. The star itself, an ultra-cool dwarf, would appear about three times larger than our own sun does in Earth's skies.\n\nThe TRAPPIST-1 system has been revealed through observations from NASA's Spitzer Space Telescope and the ground-based TRAPPIST (TRAnsiting Planets and PlanetesImals Small Telescope) telescope, as well as other ground-based observatories. The system was named for the TRAPPIST telescope.\n\nNASA's Jet Propulsion Laboratory, Pasadena, California, manages the Spitzer Space Telescope mission for NASA's Science Mission Directorate, Washington. Science operations are conducted at the Spitzer Science Center at Caltech in Pasadena. Spacecraft operations are based at Lockheed Martin Space Systems Company, Littleton, Colorado. Data are archived at the Infrared Science Archive housed at Caltech/IPAC. Caltech manages JPL for NASA. "
# ReferenceURL: b'http://www.spitzer.caltech.edu/images/6274-ssc2017-01c-Surface-of-TRAPPIST-1f'
# Credit: b'NASA/JPL-Caltech/T. Pyle (IPAC)'
# Date: b'2017-02-22'
# ID: b'ssc2017-01c'
# Type: b'Artwork'
# Publisher: b'Spitzer Science Center'
# PublisherID: b'spitzer'
# ResourceID: b'ssc2017-01c.jpg'
# ResourceURL: b'http://www.spitzer.caltech.edu/uploaded_files/images/0010/9932/ssc201
# '''