From e7b76254534534d8cbcddc9e84b7ca57ab869c7b Mon Sep 17 00:00:00 2001
From: Michael Herzberg <michael@mherzberg.de>
Date: Thu, 9 Aug 2018 14:20:01 +0100
Subject: [PATCH] Added database documentation.

---
 database/README.md                            | 134 ++++++++++++++++++
 database/config/my.cnf                        |  61 ++++++++
 .../queries/get_added_content_scripts.sql     |  16 +++
 database/queries/get_added_permissions.sql    |  16 +++
 database/schemas/category.sql                 |  41 ++++++
 database/schemas/cdnjs.sql                    |  54 +++++++
 database/schemas/content_script_url.sql       |  40 ++++++
 database/schemas/crx.sql                      |  42 ++++++
 database/schemas/crxfile.sql                  |  48 +++++++
 database/schemas/extension.sql                |  54 +++++++
 database/schemas/libdet.sql                   |  56 ++++++++
 database/schemas/permission.sql               |  40 ++++++
 database/schemas/reply.sql                    |  46 ++++++
 database/schemas/reply_comment.sql            |  39 +++++
 database/schemas/review.sql                   |  46 ++++++
 database/schemas/review_comment.sql           |  39 +++++
 database/schemas/status.sql                   |  42 ++++++
 database/schemas/support.sql                  |  46 ++++++
 database/schemas/support_comment.sql          |  39 +++++
 database/scripts/mariabackup-full             |   6 +
 database/scripts/mariabackup-inc              |  12 ++
 database/scripts/mariabackup-schemas          |  13 ++
 database/scripts/showgrants                   |   5 +
 database/views/extension_most_recent.sql      |  43 ++++++
 .../views/extension_most_recent_small.sql     |  43 ++++++
 .../extension_most_recent_until_date.sql      |  46 ++++++
 .../views/extension_second_most_recent.sql    |  43 ++++++
 ...xtension_second_most_recent_until_date.sql |  46 ++++++
 database/views/extension_small.sql            |  43 ++++++
 database/views/extension_update.sql           |  43 ++++++
 queries/get_added_content_scripts.sql         |  52 -------
 queries/get_added_permissions.sql             |  53 -------
 32 files changed, 1242 insertions(+), 105 deletions(-)
 create mode 100644 database/README.md
 create mode 100644 database/config/my.cnf
 create mode 100644 database/queries/get_added_content_scripts.sql
 create mode 100644 database/queries/get_added_permissions.sql
 create mode 100644 database/schemas/category.sql
 create mode 100644 database/schemas/cdnjs.sql
 create mode 100644 database/schemas/content_script_url.sql
 create mode 100644 database/schemas/crx.sql
 create mode 100644 database/schemas/crxfile.sql
 create mode 100644 database/schemas/extension.sql
 create mode 100644 database/schemas/libdet.sql
 create mode 100644 database/schemas/permission.sql
 create mode 100644 database/schemas/reply.sql
 create mode 100644 database/schemas/reply_comment.sql
 create mode 100644 database/schemas/review.sql
 create mode 100644 database/schemas/review_comment.sql
 create mode 100644 database/schemas/status.sql
 create mode 100644 database/schemas/support.sql
 create mode 100644 database/schemas/support_comment.sql
 create mode 100755 database/scripts/mariabackup-full
 create mode 100755 database/scripts/mariabackup-inc
 create mode 100755 database/scripts/mariabackup-schemas
 create mode 100755 database/scripts/showgrants
 create mode 100644 database/views/extension_most_recent.sql
 create mode 100644 database/views/extension_most_recent_small.sql
 create mode 100644 database/views/extension_most_recent_until_date.sql
 create mode 100644 database/views/extension_second_most_recent.sql
 create mode 100644 database/views/extension_second_most_recent_until_date.sql
 create mode 100644 database/views/extension_small.sql
 create mode 100644 database/views/extension_update.sql
 delete mode 100644 queries/get_added_content_scripts.sql
 delete mode 100644 queries/get_added_permissions.sql

diff --git a/database/README.md b/database/README.md
new file mode 100644
index 0000000..aac573f
--- /dev/null
+++ b/database/README.md
@@ -0,0 +1,134 @@
+# Introduction
+
+The extension crawler downloads all metadata and extension files into tar files.
+This is great for archival, but not so great for analyzing the data. The crawler
+therefore also supports inserting all newly crawled information into a MariaDB
+database.  Additionally, there exists a script to regenerate the database from
+old tar files.
+
+
+# Setting up the database
+
+## Hardware requirements
+
+The database is meant to be setup on a (old) PC, although it should also work
+with common cloud offerings.
+
+The amount of data that the database needs to handle grows over time. Currently,
+containing ~18 months worth of data, the database requires ~150GB of space.
+
+It is recommended to have at least 16GB of RAM to keep the indices available;
+less RAM might work, more RAM will certainly speed queries up. It is also good
+to have at least 16GB of swap; while this detrimental to the performance of
+MariaDB, it is often better than it being killed by the OS.
+
+For storage, it is beneficial to have at least one HDD and one SSD, as the
+database workload can be split into sequential and random IO.
+
+
+## Configuration
+
+A commented configuration file for MariaDB can be found in `config/my.cnf`.
+Configuration options such as pool size and storage locations will need to be
+adjusted.
+
+## Table schemas
+
+To set up the tables and schemas, make sure that you have the credentials for
+root in your `~/.my.cnf` file, and execute the following:
+```bash
+mysql -e "create database extensions;"
+for f in schemas/*.sql; do mysql extensions < $f; done
+for f in views/*.sql; do mysql extensions < $f; done
+```
+
+# Maintaining the database
+
+## Memory consumption
+
+MariaDB will, at times, use much more memory than specified for the pool size --
+100GB with a pool size of 4GB is certainly possible while regenerating the data.
+In these cases, the database should be restarted. The crawler and regeneration
+script will retry their database operations by default for around one hour.
+
+## Backup
+
+Regenerating the whole data set can take days, if not weeks, so even though all
+data can be restored, having a backup speeds up recovery. For this purpose, the
+MariaDB binary log is enabled to allow physical backups, which are much faster
+than logical backups for our case. The folder `scripts/` contains scripts to do
+full and incremental backups, as well as scripts to backup the schemas and users
+(including permissions and hashed passwords).
+
+# Regenerating extension data
+
+When the crawler is changed to extract more or different data from the
+extensions, one will probably want to regenerate all data, i.e., ask the crawler
+to go through all existing tar files and re-extract the already downloaded data.
+In order to do so, the `create-db` or `sge/create-db.sh` (for HPCs) can be used.
+More information can be found when calling these scripts with `--help`.
+
+# Using the data set
+
+## Example queries
+
+For more (commented) queries, see the `queries/` folder.
+
+- ```sql
+select extid,crx_etag,count(filename) from extension_most_recent_small join crxfile using (crx_etag) where filename like '%.js' group by extid,crx_etag limit 10;
+```
+This query will print the number of JavaScript files per extension.
+
+## Table schemas
+
+All schema files can be found in the `schemas/` folder.
+
+| Table name | Description |
+| --- | --- |
+| extension | General extension metadata from the store pages. One row per \
+extension and crawldate (!). If you are only interested in the most recent \
+*view* of the Chrome Web Store, use the `extension_most_recent` view. For \
+testing your queries, suffix either table/view with *\_small* to only get \
+roughly 1/256th of all extensions. |
+| status | The HTTP status codes for the store page and `crx` download. |
+| crx | General metadata of the extension file (the `crx` archive itself). Also \
+contains the manifest. |
+| crxfile | General metadata of the extension files, e.g., the files contained \
+in the `crx` archives (JavaScript files, etc.).|
+| category | Categories of the extensions, e.g. *productivity*, *office*, \
+or *game*. |
+| permission | Permissions found in the manifests, e.g., *webRequest*, *tab*, but also \
+host permissions such as *https://www.google.com*. |
+| content_script_url | Content script URLs found in the manifest. These are the \
+URLs where the extensions request to have a content script executed when the \
+user visits the website. |
+| libdet | Information about used libraries. For each file found in `crx` \
+archives (identified by MD5 sums), this table stores classifications of the \
+file, e.g., whether it is a certain library. |
+| review{,\_comment} | Post-metadata and posts from the review forum of an extension. |
+| support{,\_comment} | Post-metadata and posts from the support forum of an extension. |
+| reply{,\_comment} | Reply-post-metadata and posts for both the review and support forums. |
+
+## Views
+
+All views can be found in the `views/` folder.
+
+| View name | Description |
+| --- | --- |
+| extension_small | Contains only roughly 1/256th of all extensions. |
+| extension_most_recent | Instead of one row for every combination of extension \
+id and crawl date, this view only contains the rows from the most recent crawl \
+date. |
+| extension_most_recent_small | Same, but roughly only 1/256th of all extensions. |
+| extension_second_most_recent | Similar to `extension_most_recent`, but \
+contains the second-most recent entry for all extensions. This is useful for \
+investigating how extensions change. |
+| extension_{most,second_most}_recent_until_date | Parameterized query. Only \
+considers extensions crawled before a given date. Usage:  \
+```sql
+select * from (select @until_date:='2018-05-25') foo, extension_most_recent_until_date;
+``` |
+| extension_update | Selects all extension updates in the database. A row in the result represents \
+one extension update, with the date and crx_etag when we have first seen the \
+update, and the date and crx_etag when we have last seen the old version. As \
+we crawl every night, the difference should be around 24 hours on average. |
diff --git a/database/config/my.cnf b/database/config/my.cnf
new file mode 100644
index 0000000..d5d547e
--- /dev/null
+++ b/database/config/my.cnf
@@ -0,0 +1,61 @@
+[client]
+port		= 3306
+socket		= /run/mysqld/mysqld.sock
+
+[mysqld]
+port		= 3306
+socket		= /run/mysqld/mysqld.sock
+
+wait_timeout=1800
+max_connections=1000
+explicit_defaults_for_timestamp=1
+default_time_zone='+00:00'
+
+server-id	= 1
+
+expire_logs_days=8
+log-basename=master1-bin
+
+# Ideally, the MariaDB datadir resides on a HDD, as there will be a lot of sequential IO.
+# After creating a database, it is best moved to a SSD, as there will be a lot of
+# random IO. This can be done by simply moving the directory (do NOT move individual table
+# files!), e.g.: cd /hdd/mysql; mv extensions /ssd/databases/; ln -s /ssd/databases/extensions
+datadir=/hdd/mysql
+
+# When adding indices, MariaDB uses a lot of space in /tmp. If that space is not enough, the
+# used tmpdir can be moved:
+innodb_tmpdir=/ssd/innodb_tmp
+
+# The pool size is said to be around 75% of the available RAM on db-only hosts. However, current
+# versions of MariaDB seem to have serious memory leaks when doing a lot of concurrent writes.
+# Therefore, expect MariaDB to use a lot more memory, create sufficient swap to prevent killing,
+# and restart MariaDB when the usage grows too high.
+innodb_buffer_pool_size = 18G
+
+# General performance tweaks
+innodb_read_io_threads=8
+innodb_write_io_threads=8
+innodb_sort_buffer_size=67108864
+innodb_log_file_size=256M
+innodb_log_buffer_size=256M
+
+# Performance tweaks for inserts
+#innodb_flush_log_at_trx_commit=0
+#innodb_change_buffer_max_size=50
+#innodb_flush_method=O_DIRECT
+
+[mysqldump]
+quick
+max_allowed_packet = 16M
+
+[mysql]
+no-auto-rehash
+
+[myisamchk]
+key_buffer_size = 20M
+sort_buffer_size = 20M
+read_buffer = 2M
+write_buffer = 2M
+
+[mysqlhotcopy]
+interactive-timeout
diff --git a/database/queries/get_added_content_scripts.sql b/database/queries/get_added_content_scripts.sql
new file mode 100644
index 0000000..ad56642
--- /dev/null
+++ b/database/queries/get_added_content_scripts.sql
@@ -0,0 +1,16 @@
+select downloads, eu.extid, name, url, new_crx_etag
+from extension_update eu join extension e on eu.extid=e.extid and eu.first_date_with_new_crx_etag=e.date
+join content_script_url c on eu.new_crx_etag=c.crx_etag
+where
+  url in (
+    "file://*/*",
+    "http://*/*",
+    "https://*/*",
+    "*://*/*",
+    "<all_urls>"
+  )
+and
+  url not in (select url from content_script_url where crx_etag=previous_crx_etag)
+and
+  first_date_with_new_crx_etag > NOW() - INTERVAL 2 DAY
+order by downloads desc;
diff --git a/database/queries/get_added_permissions.sql b/database/queries/get_added_permissions.sql
new file mode 100644
index 0000000..d51cef8
--- /dev/null
+++ b/database/queries/get_added_permissions.sql
@@ -0,0 +1,16 @@
+select downloads, eu.extid, name, permission, new_crx_etag
+from extension_update eu join extension e on eu.extid=e.extid and eu.first_date_with_new_crx_etag=e.date
+join permission p on eu.new_crx_etag=p.crx_etag
+where
+  permission in (
+    "<all_url>",
+    "http://*/*",
+    "https://*/*",
+    "webRequest",
+    "webRequestBlocking"
+  )
+and
+  permission not in (select permission from permission where crx_etag=previous_crx_etag)
+and
+  first_date_with_new_crx_etag > NOW() - INTERVAL 2 DAY
+order by downloads desc;
diff --git a/database/schemas/category.sql b/database/schemas/category.sql
new file mode 100644
index 0000000..b91cb9b
--- /dev/null
+++ b/database/schemas/category.sql
@@ -0,0 +1,41 @@
+-- MySQL dump 10.16  Distrib 10.3.8-MariaDB, for Linux (x86_64)
+--
+-- Host: localhost    Database: extensions
+-- ------------------------------------------------------
+-- Server version	10.3.8-MariaDB-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `category`
+--
+
+DROP TABLE IF EXISTS `category`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `category` (
+  `extid` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL,
+  `date` timestamp(6) NOT NULL,
+  `category_md5` varbinary(16) NOT NULL,
+  `category` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '',
+  `last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
+  PRIMARY KEY (`extid`,`date`,`category_md5`) KEY_BLOCK_SIZE=8
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2018-08-09 12:31:29
diff --git a/database/schemas/cdnjs.sql b/database/schemas/cdnjs.sql
new file mode 100644
index 0000000..5ba4902
--- /dev/null
+++ b/database/schemas/cdnjs.sql
@@ -0,0 +1,54 @@
+-- MySQL dump 10.16  Distrib 10.3.8-MariaDB, for Linux (x86_64)
+--
+-- Host: localhost    Database: extensions
+-- ------------------------------------------------------
+-- Server version	10.3.8-MariaDB-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `cdnjs`
+--
+
+DROP TABLE IF EXISTS `cdnjs`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `cdnjs` (
+  `path` varchar(512) COLLATE utf8mb4_unicode_ci NOT NULL,
+  `typ` enum('AS_IS','NORMALIZED','DECOMPRESSED','DECOMPRESSED_NORMALIZED') COLLATE utf8mb4_unicode_ci NOT NULL,
+  `md5` varbinary(16) NOT NULL,
+  `filename` varchar(253) /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `sha1` varbinary(20) DEFAULT NULL,
+  `sha256` varbinary(32) DEFAULT NULL,
+  `simhash` varbinary(64) DEFAULT NULL,
+  `size` bigint(20) DEFAULT NULL,
+  `loc` bigint(20) DEFAULT NULL,
+  `description` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `encoding` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `mimetype` varchar(126) /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `add_date` timestamp(6) NULL DEFAULT NULL,
+  `library` varchar(254) /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `version` varchar(30) /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `mimetype_detail` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
+  PRIMARY KEY (`path`,`typ`),
+  KEY `cdnjs_md5_typ` (`md5`,`typ`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2018-08-09 12:31:29
diff --git a/database/schemas/content_script_url.sql b/database/schemas/content_script_url.sql
new file mode 100644
index 0000000..7a0b970
--- /dev/null
+++ b/database/schemas/content_script_url.sql
@@ -0,0 +1,40 @@
+-- MySQL dump 10.16  Distrib 10.3.8-MariaDB, for Linux (x86_64)
+--
+-- Host: localhost    Database: extensions
+-- ------------------------------------------------------
+-- Server version	10.3.8-MariaDB-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `content_script_url`
+--
+
+DROP TABLE IF EXISTS `content_script_url`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `content_script_url` (
+  `crx_etag` varchar(44) COLLATE utf8mb4_unicode_ci NOT NULL,
+  `url_md5` varbinary(16) NOT NULL,
+  `url` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '',
+  `last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
+  PRIMARY KEY (`crx_etag`,`url_md5`) KEY_BLOCK_SIZE=8
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2018-08-09 12:31:29
diff --git a/database/schemas/crx.sql b/database/schemas/crx.sql
new file mode 100644
index 0000000..3c3c493
--- /dev/null
+++ b/database/schemas/crx.sql
@@ -0,0 +1,42 @@
+-- MySQL dump 10.16  Distrib 10.3.8-MariaDB, for Linux (x86_64)
+--
+-- Host: localhost    Database: extensions
+-- ------------------------------------------------------
+-- Server version	10.3.8-MariaDB-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `crx`
+--
+
+DROP TABLE IF EXISTS `crx`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `crx` (
+  `crx_etag` varchar(44) COLLATE utf8mb4_unicode_ci NOT NULL,
+  `filename` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '',
+  `size` int(11) NOT NULL,
+  `publickey` blob NOT NULL,
+  `manifest` longtext /*!100301 COMPRESSED*/ CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '',
+  `last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
+  PRIMARY KEY (`crx_etag`) KEY_BLOCK_SIZE=8
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2018-08-09 12:31:29
diff --git a/database/schemas/crxfile.sql b/database/schemas/crxfile.sql
new file mode 100644
index 0000000..4d994ca
--- /dev/null
+++ b/database/schemas/crxfile.sql
@@ -0,0 +1,48 @@
+-- MySQL dump 10.16  Distrib 10.3.8-MariaDB, for Linux (x86_64)
+--
+-- Host: localhost    Database: extensions
+-- ------------------------------------------------------
+-- Server version	10.3.8-MariaDB-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `crxfile`
+--
+
+DROP TABLE IF EXISTS `crxfile`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `crxfile` (
+  `crx_etag` varchar(44) COLLATE utf8mb4_unicode_ci NOT NULL,
+  `path` varchar(512) COLLATE utf8mb4_unicode_ci NOT NULL,
+  `typ` enum('AS_IS','NORMALIZED','DECOMPRESSED','DECOMPRESSED_NORMALIZED') COLLATE utf8mb4_unicode_ci NOT NULL,
+  `md5` varbinary(16) DEFAULT NULL,
+  `filename` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `sha1` varbinary(20) DEFAULT NULL,
+  `sha256` varbinary(32) DEFAULT NULL,
+  `simhash` varbinary(64) DEFAULT NULL,
+  `mimetype` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `mimetype_detail` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
+  PRIMARY KEY (`crx_etag`,`path`,`typ`),
+  KEY `crxfile_md5_typ` (`md5`,`typ`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2018-08-09 12:31:29
diff --git a/database/schemas/extension.sql b/database/schemas/extension.sql
new file mode 100644
index 0000000..2933dfb
--- /dev/null
+++ b/database/schemas/extension.sql
@@ -0,0 +1,54 @@
+-- MySQL dump 10.16  Distrib 10.3.8-MariaDB, for Linux (x86_64)
+--
+-- Host: localhost    Database: extensions
+-- ------------------------------------------------------
+-- Server version	10.3.8-MariaDB-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `extension`
+--
+
+DROP TABLE IF EXISTS `extension`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `extension` (
+  `extid` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL,
+  `date` timestamp(6) NOT NULL,
+  `name` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `version` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `description` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `downloads` int(11) DEFAULT NULL,
+  `rating` double DEFAULT NULL,
+  `ratingcount` int(11) DEFAULT NULL,
+  `fulldescription` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `developer` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `itemcategory` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `crx_etag` varchar(44) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `lastupdated` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
+  PRIMARY KEY (`extid`,`date`) KEY_BLOCK_SIZE=8,
+  KEY `extension_crx_etag` (`crx_etag`),
+  KEY `extension_date` (`date`),
+  KEY `extension_date_extid` (`date`,`extid`),
+  KEY `extension_extid_crx_etag` (`extid`,`crx_etag`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2018-08-09 12:31:29
diff --git a/database/schemas/libdet.sql b/database/schemas/libdet.sql
new file mode 100644
index 0000000..9cb7119
--- /dev/null
+++ b/database/schemas/libdet.sql
@@ -0,0 +1,56 @@
+-- MySQL dump 10.16  Distrib 10.3.8-MariaDB, for Linux (x86_64)
+--
+-- Host: localhost    Database: extensions
+-- ------------------------------------------------------
+-- Server version	10.3.8-MariaDB-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `libdet`
+--
+
+DROP TABLE IF EXISTS `libdet`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `libdet` (
+  `md5` varbinary(16) NOT NULL,
+  `typ` enum('AS_IS','NORMALIZED','DECOMPRESSED','DECOMPRESSED_NORMALIZED') COLLATE utf8mb4_unicode_ci NOT NULL,
+  `sha1` varbinary(20) DEFAULT NULL,
+  `sha256` varbinary(32) DEFAULT NULL,
+  `size` bigint(20) DEFAULT NULL,
+  `loc` bigint(20) DEFAULT NULL,
+  `description` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `encoding` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `mimetype` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `library` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `version` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `classification_type` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `detect_method` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `detect_method_details` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `evidence_start_pos` bigint(20) DEFAULT NULL,
+  `evidence_end_pos` bigint(20) DEFAULT NULL,
+  `evidence_text` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `mimetype_detail` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `mimetype_magic` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
+  PRIMARY KEY (`md5`,`typ`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2018-08-09 12:31:29
diff --git a/database/schemas/permission.sql b/database/schemas/permission.sql
new file mode 100644
index 0000000..0fbdfe9
--- /dev/null
+++ b/database/schemas/permission.sql
@@ -0,0 +1,40 @@
+-- MySQL dump 10.16  Distrib 10.3.8-MariaDB, for Linux (x86_64)
+--
+-- Host: localhost    Database: extensions
+-- ------------------------------------------------------
+-- Server version	10.3.8-MariaDB-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `permission`
+--
+
+DROP TABLE IF EXISTS `permission`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `permission` (
+  `crx_etag` varchar(44) COLLATE utf8mb4_unicode_ci NOT NULL,
+  `permission_md5` varbinary(16) NOT NULL,
+  `permission` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
+  PRIMARY KEY (`crx_etag`,`permission_md5`) KEY_BLOCK_SIZE=8
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2018-08-09 12:31:29
diff --git a/database/schemas/reply.sql b/database/schemas/reply.sql
new file mode 100644
index 0000000..9360e74
--- /dev/null
+++ b/database/schemas/reply.sql
@@ -0,0 +1,46 @@
+-- MySQL dump 10.16  Distrib 10.3.8-MariaDB, for Linux (x86_64)
+--
+-- Host: localhost    Database: extensions
+-- ------------------------------------------------------
+-- Server version	10.3.8-MariaDB-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `reply`
+--
+
+DROP TABLE IF EXISTS `reply`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `reply` (
+  `extid` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL,
+  `date` timestamp(6) NOT NULL,
+  `author` varchar(98) COLLATE utf8mb4_unicode_ci NOT NULL,
+  `commentdate` timestamp NOT NULL,
+  `displayname` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `replyto` varchar(98) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `language` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `shortauthor` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `commentmd5` varbinary(16) DEFAULT NULL,
+  `last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
+  PRIMARY KEY (`extid`,`date`,`author`,`commentdate`) KEY_BLOCK_SIZE=8
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2018-08-09 12:31:29
diff --git a/database/schemas/reply_comment.sql b/database/schemas/reply_comment.sql
new file mode 100644
index 0000000..dc1cf1e
--- /dev/null
+++ b/database/schemas/reply_comment.sql
@@ -0,0 +1,39 @@
+-- MySQL dump 10.16  Distrib 10.3.8-MariaDB, for Linux (x86_64)
+--
+-- Host: localhost    Database: extensions
+-- ------------------------------------------------------
+-- Server version	10.3.8-MariaDB-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `reply_comment`
+--
+
+DROP TABLE IF EXISTS `reply_comment`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `reply_comment` (
+  `commentmd5` varbinary(16) NOT NULL,
+  `comment` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
+  PRIMARY KEY (`commentmd5`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2018-08-09 12:31:29
diff --git a/database/schemas/review.sql b/database/schemas/review.sql
new file mode 100644
index 0000000..c7a1e5e
--- /dev/null
+++ b/database/schemas/review.sql
@@ -0,0 +1,46 @@
+-- MySQL dump 10.16  Distrib 10.3.8-MariaDB, for Linux (x86_64)
+--
+-- Host: localhost    Database: extensions
+-- ------------------------------------------------------
+-- Server version	10.3.8-MariaDB-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `review`
+--
+
+DROP TABLE IF EXISTS `review`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `review` (
+  `extid` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL,
+  `date` timestamp(6) NOT NULL,
+  `author` varchar(98) COLLATE utf8mb4_unicode_ci NOT NULL,
+  `commentdate` timestamp NOT NULL,
+  `displayname` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `rating` double DEFAULT NULL,
+  `language` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `shortauthor` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `commentmd5` varbinary(16) DEFAULT NULL,
+  `last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
+  PRIMARY KEY (`extid`,`date`,`author`,`commentdate`) KEY_BLOCK_SIZE=8
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2018-08-09 12:31:29
diff --git a/database/schemas/review_comment.sql b/database/schemas/review_comment.sql
new file mode 100644
index 0000000..dd41321
--- /dev/null
+++ b/database/schemas/review_comment.sql
@@ -0,0 +1,39 @@
+-- MySQL dump 10.16  Distrib 10.3.8-MariaDB, for Linux (x86_64)
+--
+-- Host: localhost    Database: extensions
+-- ------------------------------------------------------
+-- Server version	10.3.8-MariaDB-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `review_comment`
+--
+
+DROP TABLE IF EXISTS `review_comment`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `review_comment` (
+  `commentmd5` varbinary(16) NOT NULL,
+  `comment` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
+  PRIMARY KEY (`commentmd5`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2018-08-09 12:31:29
diff --git a/database/schemas/status.sql b/database/schemas/status.sql
new file mode 100644
index 0000000..016f397
--- /dev/null
+++ b/database/schemas/status.sql
@@ -0,0 +1,42 @@
+-- MySQL dump 10.16  Distrib 10.3.8-MariaDB, for Linux (x86_64)
+--
+-- Host: localhost    Database: extensions
+-- ------------------------------------------------------
+-- Server version	10.3.8-MariaDB-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `status`
+--
+
+DROP TABLE IF EXISTS `status`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `status` (
+  `extid` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL,
+  `date` timestamp(6) NOT NULL,
+  `crx_status` int(11) DEFAULT NULL,
+  `overview_status` int(11) DEFAULT NULL,
+  `overview_exception` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
+  PRIMARY KEY (`extid`,`date`) KEY_BLOCK_SIZE=8
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2018-08-09 12:31:29
diff --git a/database/schemas/support.sql b/database/schemas/support.sql
new file mode 100644
index 0000000..d27629a
--- /dev/null
+++ b/database/schemas/support.sql
@@ -0,0 +1,46 @@
+-- MySQL dump 10.16  Distrib 10.3.8-MariaDB, for Linux (x86_64)
+--
+-- Host: localhost    Database: extensions
+-- ------------------------------------------------------
+-- Server version	10.3.8-MariaDB-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `support`
+--
+
+DROP TABLE IF EXISTS `support`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `support` (
+  `extid` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL,
+  `date` timestamp(6) NOT NULL,
+  `author` varchar(98) COLLATE utf8mb4_unicode_ci NOT NULL,
+  `commentdate` timestamp NOT NULL,
+  `displayname` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `title` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `language` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `shortauthor` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `commentmd5` varbinary(16) DEFAULT NULL,
+  `last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
+  PRIMARY KEY (`extid`,`date`,`author`,`commentdate`) KEY_BLOCK_SIZE=8
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2018-08-09 12:31:29
diff --git a/database/schemas/support_comment.sql b/database/schemas/support_comment.sql
new file mode 100644
index 0000000..9b1a515
--- /dev/null
+++ b/database/schemas/support_comment.sql
@@ -0,0 +1,39 @@
+-- MySQL dump 10.16  Distrib 10.3.8-MariaDB, for Linux (x86_64)
+--
+-- Host: localhost    Database: extensions
+-- ------------------------------------------------------
+-- Server version	10.3.8-MariaDB-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `support_comment`
+--
+
+DROP TABLE IF EXISTS `support_comment`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `support_comment` (
+  `commentmd5` varbinary(16) NOT NULL,
+  `comment` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
+  PRIMARY KEY (`commentmd5`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2018-08-09 12:31:29
diff --git a/database/scripts/mariabackup-full b/database/scripts/mariabackup-full
new file mode 100755
index 0000000..0e0ee39
--- /dev/null
+++ b/database/scripts/mariabackup-full
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -o nounset
+set -o errexit
+
+/usr/bin/mariabackup --backup --stream=xbstream --parallel=4 --compress --compress-threads=2
diff --git a/database/scripts/mariabackup-inc b/database/scripts/mariabackup-inc
new file mode 100755
index 0000000..48bc566
--- /dev/null
+++ b/database/scripts/mariabackup-inc
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -o nounset
+set -o errexit
+
+LSN=$1
+if ! [[ "$LSN" =~ ^[0-9]+$ ]]; then
+	>&2 echo "Invalid LSN: $LSN"
+	exit 1
+fi
+
+/usr/bin/mariabackup --backup --stream=xbstream --parallel=4 --compress --compress-threads=2 --incremental-lsn=$LSN
diff --git a/database/scripts/mariabackup-schemas b/database/scripts/mariabackup-schemas
new file mode 100755
index 0000000..d06f017
--- /dev/null
+++ b/database/scripts/mariabackup-schemas
@@ -0,0 +1,13 @@
+#!/usr/bin/bash
+
+set -o errexit
+set -o nounset
+
+T=$(mktemp -d)
+for db in $(mysql -N -e "show databases" | grep -v -e "^mysql$" -e "^information_schema$" -e "^performance_schema$")
+do
+	mkdir -p $T/schemas/$db
+	mysqldump $db --no-data --single-transaction --tab=$T/schemas/$db
+done
+(cd $T; tar cz *)
+rm -r $T
diff --git a/database/scripts/showgrants b/database/scripts/showgrants
new file mode 100755
index 0000000..cd2ee2b
--- /dev/null
+++ b/database/scripts/showgrants
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -o errexit
+set -o nounset
+
+mysql "" --skip-column-names -A -e"SELECT CONCAT('SHOW GRANTS FOR ''',user,'''@''',host,''';') FROM mysql.user WHERE user<>''" | mysql "" --skip-column-names -A | sed 's/$/;/g'
diff --git a/database/views/extension_most_recent.sql b/database/views/extension_most_recent.sql
new file mode 100644
index 0000000..400493e
--- /dev/null
+++ b/database/views/extension_most_recent.sql
@@ -0,0 +1,43 @@
+-- MySQL dump 10.16  Distrib 10.3.8-MariaDB, for Linux (x86_64)
+--
+-- Host: localhost    Database: extensions
+-- ------------------------------------------------------
+-- Server version	10.3.8-MariaDB-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Final view structure for view `extension_most_recent`
+--
+
+/*!50001 DROP TABLE IF EXISTS `extension_most_recent`*/;
+/*!50001 DROP VIEW IF EXISTS `extension_most_recent`*/;
+/*!50001 SET @saved_cs_client          = @@character_set_client */;
+/*!50001 SET @saved_cs_results         = @@character_set_results */;
+/*!50001 SET @saved_col_connection     = @@collation_connection */;
+/*!50001 SET character_set_client      = utf8 */;
+/*!50001 SET character_set_results     = utf8 */;
+/*!50001 SET collation_connection      = utf8_general_ci */;
+/*!50001 CREATE ALGORITHM=UNDEFINED */
+/*!50013 DEFINER=`writer`@`%` SQL SECURITY DEFINER */
+/*!50001 VIEW `extension_most_recent` AS select `e3`.`extid` AS `extid`,`e3`.`date` AS `date`,`e3`.`name` AS `name`,`e3`.`version` AS `version`,`e3`.`description` AS `description`,`e3`.`downloads` AS `downloads`,`e3`.`rating` AS `rating`,`e3`.`ratingcount` AS `ratingcount`,`e3`.`fulldescription` AS `fulldescription`,`e3`.`developer` AS `developer`,`e3`.`itemcategory` AS `itemcategory`,`e3`.`crx_etag` AS `crx_etag`,`e3`.`lastupdated` AS `lastupdated` from (((select `e1`.`extid` AS `extid`,max(`e1`.`date`) AS `date` from `extensions`.`extension` `e1` group by `e1`.`extid`)) `e2` join `extensions`.`extension` `e3` on(`e2`.`extid` = `e3`.`extid` and `e2`.`date` = `e3`.`date`)) */;
+/*!50001 SET character_set_client      = @saved_cs_client */;
+/*!50001 SET character_set_results     = @saved_cs_results */;
+/*!50001 SET collation_connection      = @saved_col_connection */;
+
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2018-08-09 12:31:29
diff --git a/database/views/extension_most_recent_small.sql b/database/views/extension_most_recent_small.sql
new file mode 100644
index 0000000..35bf58a
--- /dev/null
+++ b/database/views/extension_most_recent_small.sql
@@ -0,0 +1,43 @@
+-- MySQL dump 10.16  Distrib 10.3.8-MariaDB, for Linux (x86_64)
+--
+-- Host: localhost    Database: extensions
+-- ------------------------------------------------------
+-- Server version	10.3.8-MariaDB-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Final view structure for view `extension_most_recent_small`
+--
+
+/*!50001 DROP TABLE IF EXISTS `extension_most_recent_small`*/;
+/*!50001 DROP VIEW IF EXISTS `extension_most_recent_small`*/;
+/*!50001 SET @saved_cs_client          = @@character_set_client */;
+/*!50001 SET @saved_cs_results         = @@character_set_results */;
+/*!50001 SET @saved_col_connection     = @@collation_connection */;
+/*!50001 SET character_set_client      = utf8 */;
+/*!50001 SET character_set_results     = utf8 */;
+/*!50001 SET collation_connection      = utf8_general_ci */;
+/*!50001 CREATE ALGORITHM=UNDEFINED */
+/*!50013 DEFINER=`writer`@`%` SQL SECURITY DEFINER */
+/*!50001 VIEW `extension_most_recent_small` AS select `e3`.`extid` AS `extid`,`e3`.`date` AS `date`,`e3`.`name` AS `name`,`e3`.`version` AS `version`,`e3`.`description` AS `description`,`e3`.`downloads` AS `downloads`,`e3`.`rating` AS `rating`,`e3`.`ratingcount` AS `ratingcount`,`e3`.`fulldescription` AS `fulldescription`,`e3`.`developer` AS `developer`,`e3`.`itemcategory` AS `itemcategory`,`e3`.`crx_etag` AS `crx_etag`,`e3`.`lastupdated` AS `lastupdated` from (((select `e1`.`extid` AS `extid`,max(`e1`.`date`) AS `date` from `extensions`.`extension` `e1` where `e1`.`extid` like 'aa%' group by `e1`.`extid`)) `e2` join `extensions`.`extension` `e3` on(`e2`.`extid` = `e3`.`extid` and `e2`.`date` = `e3`.`date`)) */;
+/*!50001 SET character_set_client      = @saved_cs_client */;
+/*!50001 SET character_set_results     = @saved_cs_results */;
+/*!50001 SET collation_connection      = @saved_col_connection */;
+
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2018-08-09 12:31:29
diff --git a/database/views/extension_most_recent_until_date.sql b/database/views/extension_most_recent_until_date.sql
new file mode 100644
index 0000000..d6796e6
--- /dev/null
+++ b/database/views/extension_most_recent_until_date.sql
@@ -0,0 +1,46 @@
+drop function if exists until_date;
+create function until_date returns datetime NO SQL DEERMINISTIC return @until_date;
+
+-- MySQL dump 10.16  Distrib 10.3.8-MariaDB, for Linux (x86_64)
+--
+-- Host: localhost    Database: extensions
+-- ------------------------------------------------------
+-- Server version	10.3.8-MariaDB-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Final view structure for view `extension_most_recent_until_date`
+--
+
+/*!50001 DROP TABLE IF EXISTS `extension_most_recent_until_date`*/;
+/*!50001 DROP VIEW IF EXISTS `extension_most_recent_until_date`*/;
+/*!50001 SET @saved_cs_client          = @@character_set_client */;
+/*!50001 SET @saved_cs_results         = @@character_set_results */;
+/*!50001 SET @saved_col_connection     = @@collation_connection */;
+/*!50001 SET character_set_client      = utf8 */;
+/*!50001 SET character_set_results     = utf8 */;
+/*!50001 SET collation_connection      = utf8_general_ci */;
+/*!50001 CREATE ALGORITHM=UNDEFINED */
+/*!50013 DEFINER=`root`@`localhost` SQL SECURITY DEFINER */
+/*!50001 VIEW `extension_most_recent_until_date` AS select `e1`.`extid` AS `extid`,`e1`.`date` AS `date`,`extensions`.`extension`.`name` AS `name`,`extensions`.`extension`.`version` AS `version`,`extensions`.`extension`.`description` AS `description`,`extensions`.`extension`.`downloads` AS `downloads`,`extensions`.`extension`.`rating` AS `rating`,`extensions`.`extension`.`ratingcount` AS `ratingcount`,`extensions`.`extension`.`fulldescription` AS `fulldescription`,`extensions`.`extension`.`developer` AS `developer`,`extensions`.`extension`.`itemcategory` AS `itemcategory`,`extensions`.`extension`.`crx_etag` AS `crx_etag`,`extensions`.`extension`.`lastupdated` AS `lastupdated`,`extensions`.`extension`.`last_modified` AS `last_modified` from (((select `extensions`.`extension`.`extid` AS `extid`,max(`extensions`.`extension`.`date`) AS `date` from `extensions`.`extension` where `extensions`.`extension`.`date` <= `until_date`() group by `extensions`.`extension`.`extid`)) `e1` join `extensions`.`extension` on(`e1`.`extid` = `extensions`.`extension`.`extid` and `e1`.`date` = `extensions`.`extension`.`date`)) */;
+/*!50001 SET character_set_client      = @saved_cs_client */;
+/*!50001 SET character_set_results     = @saved_cs_results */;
+/*!50001 SET collation_connection      = @saved_col_connection */;
+
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2018-08-09 12:31:29
diff --git a/database/views/extension_second_most_recent.sql b/database/views/extension_second_most_recent.sql
new file mode 100644
index 0000000..d652267
--- /dev/null
+++ b/database/views/extension_second_most_recent.sql
@@ -0,0 +1,43 @@
+-- MySQL dump 10.16  Distrib 10.3.8-MariaDB, for Linux (x86_64)
+--
+-- Host: localhost    Database: extensions
+-- ------------------------------------------------------
+-- Server version	10.3.8-MariaDB-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Final view structure for view `extension_second_most_recent`
+--
+
+/*!50001 DROP TABLE IF EXISTS `extension_second_most_recent`*/;
+/*!50001 DROP VIEW IF EXISTS `extension_second_most_recent`*/;
+/*!50001 SET @saved_cs_client          = @@character_set_client */;
+/*!50001 SET @saved_cs_results         = @@character_set_results */;
+/*!50001 SET @saved_col_connection     = @@collation_connection */;
+/*!50001 SET character_set_client      = utf8 */;
+/*!50001 SET character_set_results     = utf8 */;
+/*!50001 SET collation_connection      = utf8_general_ci */;
+/*!50001 CREATE ALGORITHM=UNDEFINED */
+/*!50013 DEFINER=`root`@`localhost` SQL SECURITY DEFINER */
+/*!50001 VIEW `extension_second_most_recent` AS select `e1`.`extid` AS `extid`,`e1`.`date` AS `date`,`extensions`.`extension`.`name` AS `name`,`extensions`.`extension`.`version` AS `version`,`extensions`.`extension`.`description` AS `description`,`extensions`.`extension`.`downloads` AS `downloads`,`extensions`.`extension`.`rating` AS `rating`,`extensions`.`extension`.`ratingcount` AS `ratingcount`,`extensions`.`extension`.`fulldescription` AS `fulldescription`,`extensions`.`extension`.`developer` AS `developer`,`extensions`.`extension`.`itemcategory` AS `itemcategory`,`extensions`.`extension`.`crx_etag` AS `crx_etag`,`extensions`.`extension`.`lastupdated` AS `lastupdated`,`extensions`.`extension`.`last_modified` AS `last_modified` from (((select `extensions`.`extension`.`extid` AS `extid`,max(`extensions`.`extension`.`date`) AS `date` from `extensions`.`extension` where !((`extensions`.`extension`.`extid`,`extensions`.`extension`.`date`) in (select `extensions`.`extension`.`extid`,max(`extensions`.`extension`.`date`) AS `date` from `extensions`.`extension` group by `extensions`.`extension`.`extid`)) group by `extensions`.`extension`.`extid`)) `e1` join `extensions`.`extension` on(`e1`.`extid` = `extensions`.`extension`.`extid` and `e1`.`date` = `extensions`.`extension`.`date`)) */;
+/*!50001 SET character_set_client      = @saved_cs_client */;
+/*!50001 SET character_set_results     = @saved_cs_results */;
+/*!50001 SET collation_connection      = @saved_col_connection */;
+
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2018-08-09 12:31:29
diff --git a/database/views/extension_second_most_recent_until_date.sql b/database/views/extension_second_most_recent_until_date.sql
new file mode 100644
index 0000000..99ce39f
--- /dev/null
+++ b/database/views/extension_second_most_recent_until_date.sql
@@ -0,0 +1,46 @@
+drop function if exists until_date;
+create function until_date returns datetime NO SQL DEERMINISTIC return @until_date;
+ 
+-- MySQL dump 10.16  Distrib 10.3.8-MariaDB, for Linux (x86_64)
+--
+-- Host: localhost    Database: extensions
+-- ------------------------------------------------------
+-- Server version	10.3.8-MariaDB-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Final view structure for view `extension_second_most_recent_until_date`
+--
+
+/*!50001 DROP TABLE IF EXISTS `extension_second_most_recent_until_date`*/;
+/*!50001 DROP VIEW IF EXISTS `extension_second_most_recent_until_date`*/;
+/*!50001 SET @saved_cs_client          = @@character_set_client */;
+/*!50001 SET @saved_cs_results         = @@character_set_results */;
+/*!50001 SET @saved_col_connection     = @@collation_connection */;
+/*!50001 SET character_set_client      = utf8 */;
+/*!50001 SET character_set_results     = utf8 */;
+/*!50001 SET collation_connection      = utf8_general_ci */;
+/*!50001 CREATE ALGORITHM=UNDEFINED */
+/*!50013 DEFINER=`root`@`localhost` SQL SECURITY DEFINER */
+/*!50001 VIEW `extension_second_most_recent_until_date` AS select `e1`.`extid` AS `extid`,`e1`.`date` AS `date`,`extensions`.`extension`.`name` AS `name`,`extensions`.`extension`.`version` AS `version`,`extensions`.`extension`.`description` AS `description`,`extensions`.`extension`.`downloads` AS `downloads`,`extensions`.`extension`.`rating` AS `rating`,`extensions`.`extension`.`ratingcount` AS `ratingcount`,`extensions`.`extension`.`fulldescription` AS `fulldescription`,`extensions`.`extension`.`developer` AS `developer`,`extensions`.`extension`.`itemcategory` AS `itemcategory`,`extensions`.`extension`.`crx_etag` AS `crx_etag`,`extensions`.`extension`.`lastupdated` AS `lastupdated`,`extensions`.`extension`.`last_modified` AS `last_modified` from (((select `extensions`.`extension`.`extid` AS `extid`,max(`extensions`.`extension`.`date`) AS `date` from `extensions`.`extension` where `extensions`.`extension`.`date` <= `until_date`() and !((`extensions`.`extension`.`extid`,`extensions`.`extension`.`date`) in (select `extensions`.`extension`.`extid`,max(`extensions`.`extension`.`date`) AS `date` from `extensions`.`extension` where `extensions`.`extension`.`date` <= `until_date`() group by `extensions`.`extension`.`extid`)) group by `extensions`.`extension`.`extid`)) `e1` join `extensions`.`extension` on(`e1`.`extid` = `extensions`.`extension`.`extid` and `e1`.`date` = `extensions`.`extension`.`date`)) */;
+/*!50001 SET character_set_client      = @saved_cs_client */;
+/*!50001 SET character_set_results     = @saved_cs_results */;
+/*!50001 SET collation_connection      = @saved_col_connection */;
+
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2018-08-09 12:31:29
diff --git a/database/views/extension_small.sql b/database/views/extension_small.sql
new file mode 100644
index 0000000..e18b275
--- /dev/null
+++ b/database/views/extension_small.sql
@@ -0,0 +1,43 @@
+-- MySQL dump 10.16  Distrib 10.3.8-MariaDB, for Linux (x86_64)
+--
+-- Host: localhost    Database: extensions
+-- ------------------------------------------------------
+-- Server version	10.3.8-MariaDB-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Final view structure for view `extension_small`
+--
+
+/*!50001 DROP TABLE IF EXISTS `extension_small`*/;
+/*!50001 DROP VIEW IF EXISTS `extension_small`*/;
+/*!50001 SET @saved_cs_client          = @@character_set_client */;
+/*!50001 SET @saved_cs_results         = @@character_set_results */;
+/*!50001 SET @saved_col_connection     = @@collation_connection */;
+/*!50001 SET character_set_client      = utf8 */;
+/*!50001 SET character_set_results     = utf8 */;
+/*!50001 SET collation_connection      = utf8_general_ci */;
+/*!50001 CREATE ALGORITHM=UNDEFINED */
+/*!50013 DEFINER=`writer`@`%` SQL SECURITY DEFINER */
+/*!50001 VIEW `extension_small` AS select `extension`.`extid` AS `extid`,`extension`.`date` AS `date`,`extension`.`name` AS `name`,`extension`.`version` AS `version`,`extension`.`description` AS `description`,`extension`.`downloads` AS `downloads`,`extension`.`rating` AS `rating`,`extension`.`ratingcount` AS `ratingcount`,`extension`.`fulldescription` AS `fulldescription`,`extension`.`developer` AS `developer`,`extension`.`itemcategory` AS `itemcategory`,`extension`.`crx_etag` AS `crx_etag`,`extension`.`lastupdated` AS `lastupdated` from `extension` where `extension`.`extid` like 'aa%' */;
+/*!50001 SET character_set_client      = @saved_cs_client */;
+/*!50001 SET character_set_results     = @saved_cs_results */;
+/*!50001 SET collation_connection      = @saved_col_connection */;
+
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2018-08-09 12:31:29
diff --git a/database/views/extension_update.sql b/database/views/extension_update.sql
new file mode 100644
index 0000000..b0f8a78
--- /dev/null
+++ b/database/views/extension_update.sql
@@ -0,0 +1,43 @@
+-- MySQL dump 10.16  Distrib 10.3.8-MariaDB, for Linux (x86_64)
+--
+-- Host: localhost    Database: extensions
+-- ------------------------------------------------------
+-- Server version	10.3.8-MariaDB-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Final view structure for view `extension_update`
+--
+
+/*!50001 DROP TABLE IF EXISTS `extension_update`*/;
+/*!50001 DROP VIEW IF EXISTS `extension_update`*/;
+/*!50001 SET @saved_cs_client          = @@character_set_client */;
+/*!50001 SET @saved_cs_results         = @@character_set_results */;
+/*!50001 SET @saved_col_connection     = @@collation_connection */;
+/*!50001 SET character_set_client      = utf8 */;
+/*!50001 SET character_set_results     = utf8 */;
+/*!50001 SET collation_connection      = utf8_general_ci */;
+/*!50001 CREATE ALGORITHM=UNDEFINED */
+/*!50013 DEFINER=`root`@`%` SQL SECURITY DEFINER */
+/*!50001 VIEW `extension_update` AS select `e3`.`extid` AS `extid`,`e3`.`first_date_with_new_crx_etag` AS `first_date_with_new_crx_etag`,`e3`.`new_crx_etag` AS `new_crx_etag`,`e3`.`last_date_with_previous_crx_etag` AS `last_date_with_previous_crx_etag`,`e4`.`crx_etag` AS `previous_crx_etag` from (((select `e1`.`extid` AS `extid`,`e1`.`date` AS `first_date_with_new_crx_etag`,`e1`.`crx_etag` AS `new_crx_etag`,max(`e2`.`date`) AS `last_date_with_previous_crx_etag` from (((select `extensions`.`extension`.`extid` AS `extid`,`extensions`.`extension`.`crx_etag` AS `crx_etag`,min(`extensions`.`extension`.`date`) AS `date` from `extensions`.`extension` where `extensions`.`extension`.`crx_etag` is not null group by `extensions`.`extension`.`extid`,`extensions`.`extension`.`crx_etag`)) `e1` join (select `extensions`.`extension`.`extid` AS `extid`,`extensions`.`extension`.`crx_etag` AS `crx_etag`,max(`extensions`.`extension`.`date`) AS `date` from `extensions`.`extension` where `extensions`.`extension`.`crx_etag` is not null group by `extensions`.`extension`.`extid`,`extensions`.`extension`.`crx_etag`) `e2` on(`e1`.`extid` = `e2`.`extid`)) where `e1`.`date` > `e2`.`date` group by `e1`.`crx_etag`)) `e3` join `extensions`.`extension` `e4` on(`e3`.`extid` = `e4`.`extid` and `e3`.`last_date_with_previous_crx_etag` = `e4`.`date`)) */;
+/*!50001 SET character_set_client      = @saved_cs_client */;
+/*!50001 SET character_set_results     = @saved_cs_results */;
+/*!50001 SET collation_connection      = @saved_col_connection */;
+
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2018-08-09 12:31:29
diff --git a/queries/get_added_content_scripts.sql b/queries/get_added_content_scripts.sql
deleted file mode 100644
index c3231c4..0000000
--- a/queries/get_added_content_scripts.sql
+++ /dev/null
@@ -1,52 +0,0 @@
-select extension_info.downloads, extension_info.extid, extension_info.name, url, crx_most_recent_and_prev.crx_etag as updated_crx
-from (
-  -- We generate a table containing every crx_etag that we crawled within the
-  -- last day and is an update, and a random date where we encountered the its
-  -- previous crx_etag. We only use it to find the crx_etag again later.
-  select
-    crx_first_date1.crx_etag,
-    crx_first_date1.extid,
-    crx_first_date1.md as most_recent_update,
-    max(crx_first_date2.md) as last_date_with_previous_version
-  from (
-    select crx_etag,extid,min(date) as md
-    from extension
-    where crx_etag is not null
-    group by crx_etag
-  ) crx_first_date1
-  inner join (
-    select extid,min(date) as md
-    from extension
-    where crx_etag is not null
-    group by crx_etag
-  ) crx_first_date2
-    on crx_first_date1.extid=crx_first_date2.extid
-  where
-    crx_first_date1.md>crx_first_date2.md
-  -- the first time we have seen the crx_etag shall be zero days in the past, rounded to days,
-  -- measured from the current date and hour
-    and
-    TIMESTAMPDIFF(DAY, crx_first_date1.md,  TIMESTAMPADD(HOUR,HOUR(UTC_TIME),UTC_DATE)) = 0
-  group by crx_first_date1.crx_etag
-) crx_most_recent_and_prev
-  inner join content_script_url
-    on crx_most_recent_and_prev.crx_etag=content_script_url.crx_etag
-  inner join extension extension_info
-    on crx_most_recent_and_prev.extid=extension_info.extid
-    and extension_info.date=most_recent_update
-    and extension_info.crx_etag=crx_most_recent_and_prev.crx_etag
-where
-  url in (
-    "http://*/*",
-    "https://*/*",
-    "*://*/*",
-    "<all_urls>"
-  )
-and
-url not in (
-  select url
-  from extension natural join content_script_url
-  where extid=extension_info.extid and date=last_date_with_previous_version
-)
-order by extension_info.downloads desc;
-
diff --git a/queries/get_added_permissions.sql b/queries/get_added_permissions.sql
deleted file mode 100644
index 8e56dca..0000000
--- a/queries/get_added_permissions.sql
+++ /dev/null
@@ -1,53 +0,0 @@
-select extension_info.downloads, extension_info.extid, extension_info.name, permission, crx_most_recent_and_prev.crx_etag as updated_crx
-from (
-  -- We generate a table containing every crx_etag that we crawled within the
-  -- last day and is an update, and a random date where we encountered the its
-  -- previous crx_etag. We only use it to find the crx_etag again later.
-  select
-    crx_first_date1.crx_etag,
-    crx_first_date1.extid,
-    crx_first_date1.md as most_recent_update,
-    max(crx_first_date2.md) as last_date_with_previous_version
-  from (
-    select crx_etag,extid,min(date) as md
-    from extension
-    where crx_etag is not null
-    group by crx_etag
-  ) crx_first_date1
-  inner join (
-    select extid,min(date) as md
-    from extension
-    where crx_etag is not null
-    group by crx_etag
-  ) crx_first_date2
-    on crx_first_date1.extid=crx_first_date2.extid
-  where
-    crx_first_date1.md>crx_first_date2.md
-  -- the first time we have seen the crx_etag shall be zero days in the past, rounded to days,
-  -- measured from the current date and hour
-    and
-    TIMESTAMPDIFF(DAY, crx_first_date1.md,  TIMESTAMPADD(HOUR,HOUR(UTC_TIME),UTC_DATE)) = 0
-  group by crx_first_date1.crx_etag
-) crx_most_recent_and_prev
-  inner join permission
-    on crx_most_recent_and_prev.crx_etag=permission.crx_etag
-  inner join extension extension_info
-    on crx_most_recent_and_prev.extid=extension_info.extid
-    and extension_info.date=most_recent_update
-    and extension_info.crx_etag=crx_most_recent_and_prev.crx_etag
-where
-  permission in (
-    "<all_url>",
-    "http://*/*",
-    "https://*/*",
-    "webRequest",
-    "webRequestBlocking"
-  )
-and
-permission not in (
-  select permission
-  from extension natural join permission
-  where extid=extension_info.extid and date=last_date_with_previous_version
-)
-order by extension_info.downloads desc;
-