BrowserSecurity
/
ExtensionCrawler
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'master' of logicalhacking.com:BrowserSecurity/ExtensionCrawler

This commit is contained in:
Achim D. Brucker 2017-07-16 14:02:40 +01:00
parent 049db03021 5e60715d98
commit 3346b1b61a
4 changed files with 23 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scripts/generate_small_db.sh
View File

@ -10,10 +10,6 @@ BASEDIR=${1:-/shared/brucker_research1/Shared/BrowserExtensions/data}
DBPATH=${2:-~/aa-ac.sqlite}
EXTENSIONCRAWLER=${3:-~/ExtensionCrawler}
FIRSTDB=$(find "$BASEDIR"/aa* -name "*.sqlite" | head -n 1)
sqlite3 "$FIRSTDB" .schema | sqlite3 "$DBPATH"
echo "Used $FIRSTDB for schema"
find "$BASEDIR"/aa* -name "*.sqlite" -exec "$EXTENSIONCRAWLER/scripts/merge_dbs.sh" "{}" "$DBPATH" \;
find "$BASEDIR"/ab* -name "*.sqlite" -exec "$EXTENSIONCRAWLER/scripts/merge_dbs.sh" "{}" "$DBPATH" \;
find "$BASEDIR"/ac* -name "*.sqlite" -exec "$EXTENSIONCRAWLER/scripts/merge_dbs.sh" "{}" "$DBPATH" \;

6
scripts/global_update.sh
View File

@ -45,11 +45,7 @@ fi
date +"* Start Creating full.sqlite Data Base (%c) using $SQLITE"
# Update full database
rm -f $ARCHIVE/db/full.sqlite
(FIRSTDB=$(find "$ARCHIVE"/data/aa* -name "*.sqlite" | head -n 1);
sqlite3 "$FIRSTDB" .schema | sqlite3 "$ARCHIVE"/db/full.sqlite;
echo "Used $FIRSTDB for schema";
find "$ARCHIVE"/data/ -name "*.sqlite" -exec "$CRAWLERHOME/scripts/merge_dbs.sh" "{}" "$ARCHIVE"/db/full.sqlite \; ;
) &> $LOGPREFIX-sqlite-full.log
find "$ARCHIVE"/data/ -name "*.sqlite" -exec "$CRAWLERHOME/scripts/merge_dbs.sh" "{}" "$ARCHIVE"/db/full.sqlite \; &> $LOGPREFIX-sqlite-full.log
if [ $? -ne "0" ]; then
echo " Creation of full.sqlite failed - see log file for details"
else

1
scripts/merge_dbs.sge
View File

@ -16,5 +16,4 @@ override_vars
set -u
find $DBDIR -name "*.sqlite" -exec bash -c "sqlite3 {} .schema | sqlite3 $OUTDBPATH" \; -quit
find $DBDIR -name "*.sqlite" -exec "$EXTENSIONCRAWLERDIR/scripts/merge_dbs.sh" "{}" "$OUTDBPATH" \;

25
scripts/merge_dbs.sh
View File

@ -7,11 +7,30 @@ if [ -z $FROM_DB ] || ! [ -f $FROM_DB ]; then
exit 1
fi
if [ -z $TO_DB ] || ! [ -f $TO_DB ]; then
echo "destination db not provided or does not exist"
if [ -z $TO_DB ]; then
echo "destination db not provided"
exit 1
fi
if ! [ -f $TO_DB ]; then
echo "Creating $TO_DB ..."
sqlite3 $FROM_DB .schema | grep -Eiv \
-e "^CREATE TABLE IF NOT EXISTS (\"|')?[a-z]+_content(\"|')?\(" \
-e "^CREATE TABLE IF NOT EXISTS (\"|')?[a-z]+_docsize(\"|')?\(" \
-e "^CREATE TABLE IF NOT EXISTS (\"|')?[a-z]+_segments(\"|')?\(" \
-e "^CREATE TABLE IF NOT EXISTS (\"|')?[a-z]+_stat(\"|')?\(" \
-e "^CREATE TABLE IF NOT EXISTS (\"|')?[a-z]+_segdir(\"|')?\(" \
| sqlite3 $TO_DB
fi
echo "Merging $FROM_DB into $TO_DB..."
sqlite3 $FROM_DB .dump | grep -v "^CREATE TABLE" | sqlite3 $TO_DB
sqlite3 $FROM_DB .dump | grep -Eiv \
-e "^CREATE TABLE" \
-e "^INSERT INTO (\"|')?sqlite_master(\"|')?" \
-e "^INSERT INTO (\"|')?[a-z]+_segments(\"|')? " \
-e "^INSERT INTO (\"|')?[a-z]+_segdir(\"|')? " \
-e "^INSERT INTO (\"|')?[a-z]+_docsize(\"|')? " \
-e "^INSERT INTO (\"|')?[a-z]+_stat(\"|')? " \
| sed -r "s/^INSERT INTO ([a-z]+)_content VALUES\([[:digit:]]+,/INSERT INTO \1 VALUES(/I" \
| sqlite3 $TO_DB