#!/usr/bin/env python3.7
#
# Copyright (C) 2016,2017 The University of Sheffield, UK
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""Python analys providing a decomposition analysis of JavaScript code in
   general and Chrome extensions in particular."""

import os
from io import StringIO
import re
import json
import zlib
from enum import Enum
from ExtensionCrawler.js_mincer import mince_js
from ExtensionCrawler.file_identifiers import get_file_identifiers, is_binary_resource
from ExtensionCrawler.dbbackend.mysql_backend import MysqlBackend
from ExtensionCrawler.util import log_info
import ExtensionCrawler.config as config


class DetectionType(Enum):
    """Enumeration for detection types."""
    # EMPTY_FILE
    FILE_SIZE = "file size"
    DEC_FILE_SIZE = "file size (after decompression)"
    STRIPPED_FILE_SIZE = "file size (after stripping)"
    DEC_STRIPPED_FILE_SIZE = "file size (after decompression and stripping)"
    # LIBRARY
    SHA1 = "sha1"
    MD5 = "md5"
    SHA1_DECOMPRESSED = "sha1 (after decompression)"
    MD5_DECOMPRESSED = "md5 (after decompression)"
    # VERY_LIKELY_LIBRARY
    SHA1_NORMALIZED = "sha1 (after normalization)"
    MD5_NORMALIZED = "md5 (after normalization)"
    SHA1_DECOMPRESSED_NORMALIZED = "sha1 (after decompression and normalization)"
    MD5_DECOMPRESSED_NORMALIZED = "md5 (after decompression and normalization)"
    FILENAME_COMMENTBLOCK = "filename and witness in comment block"
    FILENAME_CODEBLOCK = "filename and witness in code block"
    # LIKELY_LIBRARY
    COMMENTBLOCK = "witness in comment block"
    CODEBLOCK = "witness in code block"
    FILENAME = "known file name"
    URL = "known URL"
    # LIKELY_APPLICATION
    DEFAULT = "default"


class FileClassification(Enum):
    """ Enumeration for file classification"""
    EMPTY_FILE = "other (empty file)"
    METADATA = "metadata"
    LIBRARY = "known library"
    LIBRARY_RSC = "known library ressource"
    VERY_LIKELY_LIBRARY = "very likely known library"
    VERY_LIKELY_LIBRARY_RSC = "very likely known library"
    LIKELY_LIBRARY = "likely known library"
    LIKELY_LIBRARY_RSC = "likely known library ressource"
    LIKELY_APPLICATION = "likely application"
    LIKELY_APPLICATION_RSC = "likely application ressource"
    ERROR = "error"


def load_lib_identifiers():
    """Initialize identifiers for known libraries from JSON file."""
    regex_file = os.path.join(
        os.path.dirname(os.path.realpath(__file__)), '../resources/',
        'js_identifier.json')
    with open(regex_file, 'r') as json_file:
        json_content = json_file.read()
    return json.loads(json_content)


def is_ressource(file_info):
    """Chedk if file info indicates that it represents a ressource type."""
    if file_info['mimetype_magic'] is not None and is_binary_resource(file_info['mimetype_magic']):
        return True
    if file_info['dec_mimetype_magic'] is not None and is_binary_resource(file_info['dec_mimetype_magic']):
        return True
    return False


def unknown_filename_identifier():
    """Identifier for extracting version information from unknown/generic file names."""
    return re.compile(
        r'(.+)[\-\_]([0-9]{1,2}[\.|\-|\_][0-9a-z]{1,2}[\.|\-|\_][0-9a-z\-\_]*)',
        re.IGNORECASE)


def unknown_lib_identifiers():
    """List of identifiers for generic library version headers."""
    return ([
        re.compile(
            r'[\/|\/\/|\s]\*?\s?([a-zA-Z0-9\.]+)\sv?([0-9][\.|\-|\_][0-9.a-z_\\\\-]+)',
            re.IGNORECASE
        ),  # MatchType: name version, e.g. mylib v1.2.9b or mylib.anything 1.2.8
        re.compile(
            r'[\/|\/\/|\s]\*?\s?([a-zA-Z0-9\.]+)\s(?: version)\:?\s?v?([0-9][0-9.a-z_\\\\-]+)',
            re.IGNORECASE
        ),  # MatchType: name version: ver, e.g. mylib version: v1.2.9, or mylib.js version 1.2.8
        re.compile(
            r'\@*(version)\s?[\:|-]?\s?v?([0-9][\.|\-|\_][0-9.a-z_\\\\-]+)',
            re.IGNORECASE
        ),  # MatchType: version x.x.x, e.g. @version: 1.2.5 or version - 1.2.5 etc.
        re.compile(
            r'(version)[\:|\=]\s?.?([0-9]{1,2}[\.|\-|\_][0-9.a-z_\\\\-]+).?',
            re.IGNORECASE),
        re.compile(r'(.+) v([0-9]{1,2}[\.|\-|\_][0-9]{1,2}[0-9.a-z_\\\\-]*)',
                   re.IGNORECASE)
    ])


def init_file_info(path, data):
    """Initialize jsinfo record."""
    file_info = get_file_identifiers(path, data)
    file_info['lib'] = None
    file_info['version'] = None
    file_info['detectionMethod'] = None
    file_info['detectionMethodDetails'] = None
    file_info['type'] = None
    file_info['evidenceStartPos'] = None
    file_info['evidenceEndPos'] = None
    file_info['evidenceText'] = None
    file_info['lib_filename'] = None
    return file_info


def check_empty_file(file_info):
    """Check if file is empty."""
    if file_info['size'] == 0:
        file_info['detectionMethod'] = DetectionType.FILE_SIZE
        file_info['type'] = FileClassification.EMPTY_FILE
    elif file_info['dec_size'] == 0:
        file_info['detectionMethod'] = DetectionType.DEC_FILE_SIZE
        file_info['type'] = FileClassification.EMPTY_FILE
    elif file_info['size_stripped'] == 0:
        file_info['detectionMethod'] = DetectionType.STRIPPED_FILE_SIZE
        file_info['type'] = FileClassification.EMPTY_FILE
    elif file_info['dec_size_stripped'] == 0:
        file_info['detectionMethod'] = DetectionType.DEC_STRIPPED_FILE_SIZE
        file_info['type'] = FileClassification.EMPTY_FILE

    return file_info


def check_metadata(file_info):
    """Check for metadata (based on filename/path)."""
    if file_info['path'] == "manifest.json" or file_info['path'] == "_metadata/verified_contents.json":
        file_info['detectionMethod'] = DetectionType.FILENAME
        file_info['type'] = FileClassification.METADATA
    return file_info


def check_md5(con, file_info):
    """Check for known md5 hash (file content)."""
    if con is None:
        return file_info
    libver = con.get_cdnjs_info(file_info['md5'])
    if libver is None:
        return file_info
    else:
        file_info['lib'] = libver[0]
        file_info['version'] = libver[1]
        file_info['lib_filename'] = libver[2]
        if is_ressource(file_info):
            file_info['type'] = FileClassification.LIBRARY_RSC
        else:
            file_info['type'] = FileClassification.LIBRARY
        file_info['detectionMethod'] = DetectionType.MD5
        return file_info


def check_md5_decompressed(con, file_info):
    """Check for known md5 hash (decompressed file content)."""
    if con is None:
        return file_info
    if file_info['dec_md5'] is not None:
        libver = con.get_cdnjs_info(file_info['dec_md5'])
        if libver is not None:
            file_info['lib'] = libver[0]
            file_info['version'] = libver[1]
            file_info['lib_filename'] = libver[2]
            if is_ressource(file_info):
                file_info['type'] = FileClassification.LIBRARY_RSC
            else:
                file_info['type'] = FileClassification.LIBRARY
            file_info['detectionMethod'] = DetectionType.MD5_DECOMPRESSED
    return file_info


def check_md5_normalized(con, file_info):
    """Check for known md5 hash (normalized file content)."""
    if con is None:
        return file_info
    if file_info['normalized_md5'] is None:
        return file_info
    elif file_info['normalized_size'] == 0:
        return file_info
    else:
        libver = con.get_cdnjs_info(file_info['normalized_md5'])
        if libver is None:
            return file_info
        else:
            file_info['lib'] = libver[0]
            file_info['version'] = libver[1]
            file_info['lib_filename'] = libver[2]
            if is_ressource(file_info):
                file_info['type'] = FileClassification.VERY_LIKELY_LIBRARY_RSC
            else:
                file_info['type'] = FileClassification.VERY_LIKELY_LIBRARY
            file_info['detectionMethod'] = DetectionType.MD5_NORMALIZED
            return file_info


def check_md5_decompressed_normalized(con, file_info):
    """Check for known md5 hash (decompressed normalized file content)."""
    if con is None:
        return file_info
    if file_info['dec_normalized_md5'] is None:
        return file_info
    elif file_info['dec_normalized_size'] == 0:
        return file_info
    else:
        libver = con.get_cdnjs_info(file_info['dec_normalized_md5'])
        if libver is None:
            return file_info
        else:
            file_info['lib'] = libver[0]
            file_info['version'] = libver[1]
            file_info['lib_filename'] = libver[2]
            if is_ressource(file_info):
                file_info['type'] = FileClassification.VERY_LIKELY_LIBRARY_RSC
            else:
                file_info['type'] = FileClassification.VERY_LIKELY_LIBRARY
            file_info[
                'detectionMethod'] = DetectionType.MD5_DECOMPRESSED_NORMALIZED
            return file_info


def check_filename(file_info):
    """Check for known filename and typical library filename patterns."""
    # TODO
    return file_info


def check_data_blocks(file_info, str_data):
    """Check for known pattern in data (comment or code) blocks."""
    # TODO
    with StringIO(str_data) as str_obj:
        for block in mince_js(str_obj, single_line_comments_block=True):
            if block.is_comment():
                pass  # TODO
            else:
                pass  # TODO
    return []


def analyse_checksum(zipfile, js_file, js_info):
    """Check for known md5 hashes (file content)."""
    json_data = load_lib_identifiers()
    for lib in json_data:
        for info in json_data[lib]:
            if info == 'sha1':
                for lib_file in json_data[lib]['sha1']:
                    if lib_file['sha1'].lower() == js_info['sha1'].hex():
                        js_info['lib'] = lib
                        js_info['version'] = lib_file['version']
                        js_info['type'] = FileClassification.LIBRARY
                        js_info['detectionMethod'] = DetectionType.SHA1,
                        if 'comment' in lib_file:
                            js_info['detectionMethodDetails'] = lib_file[
                                'comment']
                        return [js_info]
            if info == 'md5':
                for lib_file in json_data[lib]['md5']:
                    if lib_file['md5'].lower() == js_info['md5'].hex():
                        js_info['lib'] = lib
                        js_info['version'] = lib_file['version']
                        js_info['type'] = FileClassification.LIBRARY
                        js_info['detectionMethod'] = DetectionType.MD5
                        if 'comment' in lib_file:
                            js_info['detectionMethodDetails'] = lib_file[
                                'comment']
                        return [js_info]
    return None


def analyse_known_filename(zipfile, js_file, js_info):
    """Check for known file name patterns."""
    libs = list()
    for lib, regex in load_lib_identifiers().items():
        if 'filename' in regex:
            if zipfile is not None:
                filename = js_file.filename
            else:
                filename = js_file
            filename_matched = re.search(regex['filename'], filename,
                                         re.IGNORECASE)
            if filename_matched:
                js_info['lib'] = lib
                js_info['version'] = filename_matched.group(2)
                js_info['type'] = FileClassification.LIBRARY
                js_info['detectionMethod'] = DetectionType.FILENAME
                js_info['detectionMethodDetails'] = regex['filename']
                libs.append(js_info)
    return libs


def analyse_generic_filename(zipfile, js_file, js_info):
    """Check for generic file name patterns."""
    libs = list()
    if zipfile is not None:
        filename = js_file.filename
    else:
        filename = js_file

    unknown_filename_match = unknown_filename_identifier().search(filename)
    if unknown_filename_match:
        js_info['lib'] = os.path.basename(
            unknown_filename_match.group(1)).replace('.js', '').replace(
                '.min', '')
        js_info['version'] = unknown_filename_match.group(2)
        js_info['type'] = FileClassification.LIKELY_LIBRARY
        js_info['detectionMethod'] = DetectionType.FILENAME
        libs.append(js_info)
    return libs


def analyse_filename(zipfile, js_file, js_info):
    """Check for file name patterns of libraries (known and generic as fall back)`"""
    res = analyse_known_filename(zipfile, js_file, js_info)
    if not res:
        res = analyse_generic_filename(zipfile, js_file, js_info)
    return res


def analyse_comment_known_libs(zipfile, js_file, js_info, comment):
    """Search for library specific identifiers in comment block."""
    libs = list()
    if zipfile is not None:
        filename = js_file.filename
    else:
        filename = js_file
    for lib, regex in load_lib_identifiers().items():
        if 'filecontent' in regex:
            for unkregex in regex['filecontent']:
                unkown_lib_matched = unkregex.finditer(comment.content)
                for match in unkown_lib_matched:
                    js_info['lib'] = lib
                    js_info['version'] = match.group(2)
                    js_info['detectionMethod'] = DetectionType.COMMENTBLOCK
                    js_info['detectionMethodDetails'] = unkregex
                    js_info['type'] = FileClassification.LIBRARY
                    libs.append(js_info)
    return libs


def analyse_comment_generic_libs(zipfile, js_file, js_info, comment):
    """Search for generic identifiers in comment block."""
    libs = list()
    if zipfile is not None:
        filename = js_file.filename
    else:
        filename = js_file

    for unkregex in unknown_lib_identifiers():
        unkown_lib_matched = unkregex.finditer(comment.content)
        for match in unkown_lib_matched:
            js_info['lib'] = ((os.path.basename(filename)).replace(
                '.js', '')).replace('.min', '')
            js_info['version'] = match.group(2)
            js_info['detectionMethod'] = DetectionType.COMMENTBLOCK
            js_info['detectionMethodDetails'] = unkregex
            js_info['type'] = FileClassification.LIKELY_LIBRARY
            libs.append(js_info)
    return libs


def decompose_js(path_or_zipfileobj, use_db=True):
    if use_db:
        with MysqlBackend(
                None,
                read_default_file=config.const_mysql_config_file(),
                charset='utf8mb4',
                compress=True) as con:
            return decompose_js_with_connection(path_or_zipfileobj, con)
    else:
        return decompose_js_with_connection(path_or_zipfileobj, None)


def merge_filename_and_data_info(file_filename_info, info_data_blocks):
    """Merge file information based on filename heuristics and data block analysis."""
    # TODO
    return info_data_blocks


def decompose_js_with_connection(path_or_zipfileobj, con):
    """JavaScript decomposition analysis for extensions."""
    zipfile = None
    inventory = []
    if isinstance(path_or_zipfileobj, str):
        path_list = [path_or_zipfileobj]
    else:
        zipfile = path_or_zipfileobj
        path_list = list(
            filter(lambda x: os.path.basename(x.filename) != "",
                   zipfile.infolist()))

    for path_or_zipentry in path_list:

        if zipfile is not None:
            with zipfile.open(path_or_zipentry) as js_file_obj:
                data = js_file_obj.read()
            path = path_or_zipentry.filename
        else:
            with open(path_or_zipentry, mode='rb') as js_file_obj:
                data = js_file_obj.read()
            path = path_or_zipentry

        file_info = init_file_info(path, data)

        file_info = check_empty_file(file_info)
        if not file_info['detectionMethod'] is None:
            inventory.append(file_info)
            continue
        file_info = check_metadata(file_info)
        if not file_info['detectionMethod'] is None:
            inventory.append(file_info)
            continue
        file_info = check_md5(con, file_info)
        if not file_info['detectionMethod'] is None:
            inventory.append(file_info)
            continue
        file_info = check_md5_decompressed(con, file_info)
        if not file_info['detectionMethod'] is None:
            inventory.append(file_info)
            continue
        file_info = check_md5_normalized(con, file_info)
        if not file_info['detectionMethod'] is None:
            inventory.append(file_info)
            continue
        file_info = check_md5_decompressed_normalized(con, file_info)
        if not file_info['detectionMethod'] is None:
            inventory.append(file_info)
            continue

        file_filename_info = check_filename(file_info)
        if file_info['detectionMethod'] is None:
            if not file_info['dec_encoding'] is None:
                try:
                    with zlib.decompressobj(zlib.MAX_WBITS | 16) as dec:
                        dec_data = dec.decompress(data,
                                                  100 * file_info['size'])
                    str_data = dec_data.decode(file_info['dec_encoding'])
                    del dec_data
                except Exception:
                    log_info("Exception during data decoding (decompressed) for entry " + file_info['filename'], 3)
                    str_data = ''
            elif file_info["encoding"] is not None:
                try:
                    str_data = data.decode(file_info['encoding'])
                except Exception:
                    log_info("Exception during data decoding for entry " + file_info['filename'], 3)
                    str_data = ''
            else:
                str_data = ''

            info_data_blocks = check_data_blocks(file_info, str_data)
        else:
            info_data_blocks = None

        if info_data_blocks:
            inventory = inventory + merge_filename_and_data_info(
                file_filename_info, info_data_blocks)
            continue
        else:
            file_info = file_filename_info

        # if no library could be detected, we report the JavaScript file as 'application'.
        if file_info['detectionMethod'] is None:
            file_info['lib'] = None
            file_info['version'] = None
            file_info['detectionMethod'] = DetectionType.DEFAULT
            if is_ressource(file_info):
                file_info['type'] = FileClassification.LIKELY_APPLICATION_RSC
            else:
                file_info['type'] = FileClassification.LIKELY_APPLICATION
            inventory.append(file_info)

    return inventory