BrowserSecurity
/
ExtensionCrawler
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Compare commits

base: BrowserSecurity:dc0b2b6caaab7fe305a170c3353b8721fc3963ef
Branches Tags
BrowserSecurity:master
BrowserSecurity:production
BrowserSecurity:mixed_forums
BrowserSecurity:tarcat
BrowserSecurity:indexed_tarfile
BrowserSecurity:parallel_forums
BrowserSecurity:last_sqlite3
...
compare: BrowserSecurity:16c13a4cfb1fc0993b4b0127fa0c852ddf6071c7
Branches Tags
BrowserSecurity:master
BrowserSecurity:production
BrowserSecurity:mixed_forums
BrowserSecurity:tarcat
BrowserSecurity:indexed_tarfile
BrowserSecurity:parallel_forums
BrowserSecurity:last_sqlite3

3 Commits
dc0b2b6caa ... 16c13a4cfb

Author SHA1 Message Date
Achim D. Brucker 16c13a4cfb Merge branch 'master' into production 2020-03-28 20:54:46 +00:00
Achim D. Brucker 9f56feb870 Use datatime instead of timestamp to avoid problems with Daylight Saving Times. 2020-03-28 12:12:55 +00:00
Achim D. Brucker 671395d9d1 Reduced number of parallel downloads to reduce memory footprint. 2020-03-22 21:22:34 +00:00
16 changed files with 26 additions and 26 deletions
Show Stats Expand all files Collapse all files

4
database/schemas/category.sql
View File

@ -22,10 +22,10 @@ DROP TABLE IF EXISTS `category`;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `category` (
`extid` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL,
`date` timestamp(6) NOT NULL,
`date` datetime(6) NOT NULL,
`category_md5` varbinary(16) NOT NULL,
`category` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '',
`last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
`last_modified` datetime NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
PRIMARY KEY (`extid`,`date`,`category_md5`) KEY_BLOCK_SIZE=8
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
/*!40101 SET character_set_client = @saved_cs_client */;

4
database/schemas/cdnjs.sql
View File

@ -33,11 +33,11 @@ CREATE TABLE `cdnjs` (
`description` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`encoding` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`mimetype` varchar(126) /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`add_date` timestamp(6) NULL DEFAULT NULL,
`add_date` datetime(6) NULL DEFAULT NULL,
`library` varchar(254) /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`version` varchar(30) /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`mimetype_detail` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
`last_modified` datetime NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
PRIMARY KEY (`path`,`typ`),
KEY `cdnjs_md5_typ` (`md5`,`typ`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';

2
database/schemas/content_script_url.sql
View File

@ -24,7 +24,7 @@ CREATE TABLE `content_script_url` (
`crx_etag` varchar(44) COLLATE utf8mb4_unicode_ci NOT NULL,
`url_md5` varbinary(16) NOT NULL,
`url` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '',
`last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
`last_modified` datetime NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
PRIMARY KEY (`crx_etag`,`url_md5`) KEY_BLOCK_SIZE=8
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
/*!40101 SET character_set_client = @saved_cs_client */;

2
database/schemas/crx.sql
View File

@ -26,7 +26,7 @@ CREATE TABLE `crx` (
`size` int(11) NOT NULL,
`publickey` blob NOT NULL,
`manifest` longtext /*!100301 COMPRESSED*/ CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '',
`last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
`last_modified` datetime NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
PRIMARY KEY (`crx_etag`) KEY_BLOCK_SIZE=8
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
/*!40101 SET character_set_client = @saved_cs_client */;

2
database/schemas/crxfile.sql
View File

@ -31,7 +31,7 @@ CREATE TABLE `crxfile` (
`simhash` varbinary(64) DEFAULT NULL,
`mimetype` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`mimetype_detail` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
`last_modified` datetime NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
PRIMARY KEY (`crx_etag`,`path`,`typ`),
KEY `crxfile_md5_typ` (`md5`,`typ`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';

4
database/schemas/extension.sql
View File

@ -22,7 +22,7 @@ DROP TABLE IF EXISTS `extension`;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `extension` (
`extid` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL,
`date` timestamp(6) NOT NULL,
`date` datetime(6) NOT NULL,
`name` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`version` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`description` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
@ -35,7 +35,7 @@ CREATE TABLE `extension` (
`itemcategory` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`crx_etag` varchar(44) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`lastupdated` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
`last_modified` datetime NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
PRIMARY KEY (`extid`,`date`) KEY_BLOCK_SIZE=8,
KEY `extension_crx_etag` (`crx_etag`),
KEY `extension_date` (`date`),

2
database/schemas/libdet.sql
View File

@ -40,7 +40,7 @@ CREATE TABLE `libdet` (
`evidence_text` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`mimetype_detail` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`mimetype_magic` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
`last_modified` datetime NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
PRIMARY KEY (`md5`,`typ`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
/*!40101 SET character_set_client = @saved_cs_client */;

2
database/schemas/permission.sql
View File

@ -24,7 +24,7 @@ CREATE TABLE `permission` (
`crx_etag` varchar(44) COLLATE utf8mb4_unicode_ci NOT NULL,
`permission_md5` varbinary(16) NOT NULL,
`permission` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
`last_modified` datetime NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
PRIMARY KEY (`crx_etag`,`permission_md5`) KEY_BLOCK_SIZE=8
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
/*!40101 SET character_set_client = @saved_cs_client */;

6
database/schemas/reply.sql
View File

@ -22,15 +22,15 @@ DROP TABLE IF EXISTS `reply`;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `reply` (
`extid` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL,
`date` timestamp(6) NOT NULL,
`date` datetime(6) NOT NULL,
`author` varchar(98) COLLATE utf8mb4_unicode_ci NOT NULL,
`commentdate` timestamp NOT NULL,
`commentdate` datetime NOT NULL,
`displayname` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`replyto` varchar(98) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`language` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`shortauthor` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`commentmd5` varbinary(16) DEFAULT NULL,
`last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
`last_modified` datetime NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
PRIMARY KEY (`extid`,`date`,`author`,`commentdate`) KEY_BLOCK_SIZE=8
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
/*!40101 SET character_set_client = @saved_cs_client */;

2
database/schemas/reply_comment.sql
View File

@ -23,7 +23,7 @@ DROP TABLE IF EXISTS `reply_comment`;
CREATE TABLE `reply_comment` (
`commentmd5` varbinary(16) NOT NULL,
`comment` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
`last_modified` datetime NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
PRIMARY KEY (`commentmd5`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
/*!40101 SET character_set_client = @saved_cs_client */;

6
database/schemas/review.sql
View File

@ -22,15 +22,15 @@ DROP TABLE IF EXISTS `review`;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `review` (
`extid` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL,
`date` timestamp(6) NOT NULL,
`date` datetime(6) NOT NULL,
`author` varchar(98) COLLATE utf8mb4_unicode_ci NOT NULL,
`commentdate` timestamp NOT NULL,
`commentdate` datetime NOT NULL,
`displayname` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`rating` double DEFAULT NULL,
`language` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`shortauthor` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`commentmd5` varbinary(16) DEFAULT NULL,
`last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
`last_modified` datetime NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
PRIMARY KEY (`extid`,`date`,`author`,`commentdate`) KEY_BLOCK_SIZE=8
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
/*!40101 SET character_set_client = @saved_cs_client */;

2
database/schemas/review_comment.sql
View File

@ -23,7 +23,7 @@ DROP TABLE IF EXISTS `review_comment`;
CREATE TABLE `review_comment` (
`commentmd5` varbinary(16) NOT NULL,
`comment` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
`last_modified` datetime NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
PRIMARY KEY (`commentmd5`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
/*!40101 SET character_set_client = @saved_cs_client */;

4
database/schemas/status.sql
View File

@ -22,11 +22,11 @@ DROP TABLE IF EXISTS `status`;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `status` (
`extid` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL,
`date` timestamp(6) NOT NULL,
`date` datetime(6) NOT NULL,
`crx_status` int(11) DEFAULT NULL,
`overview_status` int(11) DEFAULT NULL,
`overview_exception` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
`last_modified` datetime NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
PRIMARY KEY (`extid`,`date`) KEY_BLOCK_SIZE=8
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
/*!40101 SET character_set_client = @saved_cs_client */;

6
database/schemas/support.sql
View File

@ -22,15 +22,15 @@ DROP TABLE IF EXISTS `support`;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `support` (
`extid` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL,
`date` timestamp(6) NOT NULL,
`date` datetime(6) NOT NULL,
`author` varchar(98) COLLATE utf8mb4_unicode_ci NOT NULL,
`commentdate` timestamp NOT NULL,
`commentdate` datetime NOT NULL,
`displayname` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`title` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`language` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`shortauthor` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`commentmd5` varbinary(16) DEFAULT NULL,
`last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
`last_modified` datetime NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
PRIMARY KEY (`extid`,`date`,`author`,`commentdate`) KEY_BLOCK_SIZE=8
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
/*!40101 SET character_set_client = @saved_cs_client */;

2
database/schemas/support_comment.sql
View File

@ -23,7 +23,7 @@ DROP TABLE IF EXISTS `support_comment`;
CREATE TABLE `support_comment` (
`commentmd5` varbinary(16) NOT NULL,
`comment` text /*!100301 COMPRESSED*/ COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`last_modified` timestamp NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
`last_modified` datetime NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
PRIMARY KEY (`commentmd5`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci `PAGE_COMPRESSED`='ON';
/*!40101 SET character_set_client = @saved_cs_client */;

2
scripts/update/global_update.sh
View File

@ -14,7 +14,7 @@ LOG=$LOGPREFIX-global.log
date --utc +'* Start Updating Extensions Archive (%c)' | tee $LOG
# Update extensions
(singularity exec --bind /srv/:/srv/ $IMAGE crawler -p 42 -d --pystuck -a $ARCHIVE > $LOGPREFIX.log ) |& ts '%Y-%m-%dT%H:%M:%S' | tee $LOGPREFIX-stderr.log
(singularity exec --bind /srv/:/srv/ $IMAGE crawler -p 32 -d --pystuck -a $ARCHIVE > $LOGPREFIX.log ) |& ts '%Y-%m-%dT%H:%M:%S' | tee $LOGPREFIX-stderr.log
date --utc +'* Update Finished (%c)' | tee -a $LOG