ExtensionCrawler/crawler

#!/usr/bin/env python3
#
# Copyright (C) 2016,2017 The University of Sheffield, UK
# 
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import os
import sys
import glob
import re
import requests
from time import sleep
from random import randint
from datetime import datetime, timezone
from ExtensionCrawler.discover import *
from ExtensionCrawler.archive import *
from ExtensionCrawler.util import *
from ExtensionCrawler.discover import *
import dateutil
import dateutil.parser


def log_summary(verbose, res):
    total = len(res)
    success = len(list(filter(lambda x: x.is_ok(), res)))
    not_authorized = len(list(filter(lambda x: x.not_authorized(), res)))
    has_exception = len(list(filter(lambda x: x.has_exception(), res)))
    raised_ddos = len(list(filter(lambda x: x.raised_google_ddos(), res)))
    not_in_store = len(list(filter(lambda x: x.not_in_store(), res)))
    not_modified = len(list(filter(lambda x: x.not_modified(), res)))
    log(verbose, "\n")
    log(verbose, "Summary:\n")
    log(verbose, "    Updated {} out of {} extensions successfully\n".format(
        str(success), str(total)))
    log(verbose,
        "    Not authorized:          {}\n".format(str(not_authorized)))
    log(verbose, "    Raised Google DDOS:      {}\n".format(str(raised_ddos)))
    log(verbose, "    Not modified archives:   {}\n".format(str(not_modified)))
    log(verbose, "    Extensions not in store: {}\n".format(str(not_in_store)))
    log(verbose,
        "    Unknown exception:       {}\n".format(str(has_exception)))


def main():
    basedir = "."
    archive_dir = os.path.join(basedir, "archive")
    conf_dir = os.path.join(basedir, "conf")
    verbose = True
    skip_discovery = True

    log(verbose, "Configuration:\n")
    log(verbose, "  Base dir:       {}\n".format(basedir))
    log(verbose, "    Archive dir:  {}\n".format(archive_dir))
    log(verbose, "    Conf. dir:    {}\n".format(conf_dir))
    log(verbose, "  Skip discovery: {}\n".format(skip_discovery))
    log(verbose, "\n".format(skip_discovery))

    forum_ext_ids = get_forum_ext_ids(conf_dir, verbose)
    existing_ids = get_existing_ids(archive_dir, verbose)
    known_ids = list(set(existing_ids) | set(forum_ext_ids))
    new_ids = []
    if not skip_discovery:
        new_ids = get_new_ids(verbose, known_ids)

    res = update_extensions(archive_dir, verbose, forum_ext_ids, existing_ids,
                            new_ids)

    log_summary(verbose, res)


main()