2017-09-20 10:37:57 +00:00
|
|
|
#!/bin/sh
|
|
|
|
# Copyright 2017 The University of Sheffield, UK
|
|
|
|
#
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
|
# You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
# limitations under the License.
|
|
|
|
|
|
|
|
|
|
|
|
bootstrap:debootstrap
|
|
|
|
OSVersion: testing
|
2017-09-25 23:06:00 +00:00
|
|
|
MirrorURL: https://deb.debian.org/debian
|
2017-09-20 10:37:57 +00:00
|
|
|
|
|
|
|
%labels
|
|
|
|
Maintainer The LogicalHacking Team (https://logicalhacking.com)
|
|
|
|
|
|
|
|
%setup
|
|
|
|
|
|
|
|
%post
|
|
|
|
|
|
|
|
###################################################################
|
|
|
|
# Add Debian unstable as a secondary (lower priority) source
|
|
|
|
# and update the data base of available packages.
|
|
|
|
cat >> /etc/apt/sources.list << EOF
|
|
|
|
deb http://ftp.us.debian.org/debian unstable main
|
|
|
|
EOF
|
|
|
|
|
|
|
|
cat > /etc/apt/preferences << EOF
|
|
|
|
Package: *
|
|
|
|
Pin: release a=testing
|
|
|
|
Pin-Priority: 900
|
|
|
|
|
|
|
|
Package: *
|
|
|
|
Pin: release a=unstable
|
|
|
|
Pin-Priority: 800
|
|
|
|
EOF
|
|
|
|
|
|
|
|
cat > /etc/apt/apt.conf.d/01norecommend << EOF
|
|
|
|
APT::Install-Recommends "0";
|
|
|
|
APT::Install-Suggests "0";
|
|
|
|
EOF
|
|
|
|
|
|
|
|
chmod go+r /etc/apt/preferences
|
|
|
|
apt-get update
|
|
|
|
###################################################################
|
|
|
|
|
2017-09-20 12:32:47 +00:00
|
|
|
###################################################################
|
|
|
|
# Configure locales
|
|
|
|
apt-get install -y locales
|
2017-09-25 21:42:29 +00:00
|
|
|
echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen
|
2017-10-04 22:17:36 +00:00
|
|
|
echo "en_GB.UTF-8 UTF-8" >> /etc/locale.gen
|
|
|
|
locale-gen
|
2017-09-25 21:42:29 +00:00
|
|
|
echo "LANG=en_US.UTF-8" > /etc/default/locale
|
2017-09-20 12:32:47 +00:00
|
|
|
###################################################################
|
|
|
|
|
2017-09-20 10:37:57 +00:00
|
|
|
###################################################################
|
2017-11-08 22:11:15 +00:00
|
|
|
# Install the core dependencies (Python 3.6 or later)
|
2017-09-20 10:37:57 +00:00
|
|
|
# from the Debian Testing repository
|
2017-10-08 16:17:41 +00:00
|
|
|
apt-get install -y python3-magic python3-crypto python3-minimal python3-pip python3-setuptools python3-mysqldb python3-jsbeautifier python3-tabulate
|
2017-09-20 10:37:57 +00:00
|
|
|
apt-get clean
|
|
|
|
apt-get install -y git
|
|
|
|
apt-get clean
|
|
|
|
rm -rf /var/lib/apt/lists/*
|
|
|
|
###################################################################
|
|
|
|
|
|
|
|
###################################################################
|
|
|
|
# Create /opt for local software (mainly cloned git repositories
|
|
|
|
# from logicalhacking.com
|
|
|
|
mkdir -p /opt
|
|
|
|
chmod 755 /opt
|
|
|
|
###################################################################
|
|
|
|
|
|
|
|
###################################################################
|
|
|
|
# Add the Extension Crawler repository, for more details, visit
|
|
|
|
# https://git.logicalhacking.com/BrowserSecurity/ExtensionCrawler
|
|
|
|
cd /opt
|
|
|
|
git clone https://git.logicalhacking.com/BrowserSecurity/ExtensionCrawler.git
|
|
|
|
cd ExtensionCrawler
|
|
|
|
git checkout production
|
|
|
|
cd ..
|
2018-04-08 23:13:25 +00:00
|
|
|
pip3 install wheel # simhash needs wheel to build properly, still works without it though
|
2017-09-20 10:37:57 +00:00
|
|
|
pip3 install --system -e ExtensionCrawler
|
|
|
|
cd /
|
|
|
|
chmod -R go+u-w /opt/ExtensionCrawler
|
|
|
|
chmod -R go+u-w /usr/local/lib/
|
|
|
|
chmod -R go+u-w /usr/local/bin/
|
|
|
|
###################################################################
|
|
|
|
|
2017-09-25 23:01:03 +00:00
|
|
|
###################################################################
|
2017-09-26 07:55:09 +00:00
|
|
|
# Clone cdnjs repository or crate link to external archive dir
|
2017-10-10 19:26:44 +00:00
|
|
|
ARCHIVE=/shared/brucker_research1/Shared/BrowserExtensions/archive
|
2017-09-26 07:55:09 +00:00
|
|
|
case ${SINGULARITY_IMAGE} in
|
|
|
|
*-cdnjs.img)
|
|
|
|
mkdir -p /opt/archive/filedb
|
|
|
|
cd /opt/archive/filedb
|
2017-09-25 23:01:03 +00:00
|
|
|
git clone https://github.com/cdnjs/cdnjs.git cdnjs-git
|
|
|
|
cd cdnjs-git
|
|
|
|
git pull
|
2017-10-10 19:26:44 +00:00
|
|
|
ln -s ${ARCHIVE}/conf . > /dev/null
|
|
|
|
ln -s ${ARCHIVE}/data > /dev/null
|
|
|
|
ln -s ${ARCHIVE}/log > /dev/null
|
2017-09-26 07:55:09 +00:00
|
|
|
;;
|
|
|
|
*)
|
|
|
|
cd /opt/
|
|
|
|
ln -s ${ARCHIVE} .
|
2017-09-25 23:01:03 +00:00
|
|
|
;;
|
|
|
|
esac
|
2017-10-10 19:26:44 +00:00
|
|
|
chmod -R go+u /opt
|
2017-09-25 23:01:03 +00:00
|
|
|
###################################################################
|
|
|
|
|
2017-09-20 10:37:57 +00:00
|
|
|
###################################################################
|
|
|
|
# Create mount/bind points for the various network drives
|
|
|
|
# on SHARC (only useful when using the Singularity image on
|
|
|
|
# the High-Performance Cluster of The University of Sheffield
|
|
|
|
mkdir /scratch
|
|
|
|
mkdir /fastdata
|
|
|
|
mkdir /data
|
|
|
|
mkdir /shared
|
|
|
|
|
|
|
|
# Create nvidia driver directories to get rid of the singularity
|
|
|
|
# warnings on sharc
|
|
|
|
mkdir /nvbin
|
|
|
|
mkdir /nvlib
|
|
|
|
chmod go+u-w /scratch /fastdata /data /shared
|
|
|
|
###################################################################
|
|
|
|
|
|
|
|
%environment
|
|
|
|
|
2017-09-26 08:13:31 +00:00
|
|
|
export EXTENSION_ARCHIVE=/opt/archive
|
|
|
|
export PATH=/opt/ExtensionCrawler/:${PATH}
|
2017-09-26 07:55:09 +00:00
|
|
|
|
2017-09-20 10:37:57 +00:00
|
|
|
# We install all python modules into the container, so we do not want
|
|
|
|
# to use any packages that the user might have installed in their home
|
|
|
|
# directory.
|
|
|
|
export PYTHONNOUSERSITE=1
|
2017-09-26 07:55:09 +00:00
|
|
|
|
2017-09-26 08:43:05 +00:00
|
|
|
%runscript
|
|
|
|
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
|
|
# this text will get copied to /singularity and will run whenever the container
|
|
|
|
# is called as an executable
|
|
|
|
usage() {
|
|
|
|
cat <<EOF
|
|
|
|
NAME
|
|
|
|
ExtensionCrawler
|
|
|
|
SYNOPSIS
|
|
|
|
ExtensionCrawler tool [tool options]
|
|
|
|
ExtensionCrawler list
|
|
|
|
ExtensionCrawler help
|
|
|
|
DESCRIPTION
|
|
|
|
A collection of utilities for downloading and analyzing browser extension
|
|
|
|
from the Chrome Web store.
|
|
|
|
ENVIRIONMENT
|
|
|
|
EXTENSION_ARCHIVE=${EXTENSION_ARCHIVE}
|
|
|
|
EOF
|
|
|
|
}
|
|
|
|
|
|
|
|
tools() {
|
|
|
|
find /opt/ExtensionCrawler -maxdepth 1 -executable -type f -exec sh -c "{} -h | head -n 1" \;
|
|
|
|
}
|
|
|
|
|
|
|
|
arg="${1:-none}"
|
|
|
|
|
|
|
|
case "$arg" in
|
|
|
|
none) usage; exit 1;;
|
|
|
|
help) usage; exit 0;;
|
|
|
|
list) tools; exit 0;;
|
|
|
|
# just try to execute it then
|
|
|
|
*) $@;;
|
|
|
|
esac
|
|
|
|
|